2024-11-19 17:50:32,585 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b 2024-11-19 17:50:32,595 main DEBUG Took 0.008329 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-19 17:50:32,596 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-19 17:50:32,596 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-19 17:50:32,597 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-19 17:50:32,598 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 17:50:32,604 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-19 17:50:32,615 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 17:50:32,616 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 17:50:32,617 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 17:50:32,617 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 17:50:32,618 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 17:50:32,618 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 17:50:32,619 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 17:50:32,620 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 17:50:32,620 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 17:50:32,620 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 17:50:32,621 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 17:50:32,621 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 17:50:32,622 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 17:50:32,622 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 17:50:32,623 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 17:50:32,623 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 17:50:32,624 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 17:50:32,624 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 17:50:32,625 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 17:50:32,625 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 17:50:32,626 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 17:50:32,626 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 17:50:32,627 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 17:50:32,627 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 17:50:32,628 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 17:50:32,628 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-19 17:50:32,630 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 17:50:32,632 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-19 17:50:32,634 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-19 17:50:32,635 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-19 17:50:32,637 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-19 17:50:32,637 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-19 17:50:32,648 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-19 17:50:32,652 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-19 17:50:32,654 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-19 17:50:32,655 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-19 17:50:32,656 main DEBUG createAppenders(={Console}) 2024-11-19 17:50:32,657 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b initialized 2024-11-19 17:50:32,658 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b 2024-11-19 17:50:32,658 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b OK. 2024-11-19 17:50:32,659 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-19 17:50:32,659 main DEBUG OutputStream closed 2024-11-19 17:50:32,660 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-19 17:50:32,660 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-19 17:50:32,661 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@37d4349f OK 2024-11-19 17:50:32,750 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-19 17:50:32,752 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-19 17:50:32,754 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-19 17:50:32,755 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-19 17:50:32,755 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-19 17:50:32,756 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-19 17:50:32,756 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-19 17:50:32,757 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-19 17:50:32,757 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-19 17:50:32,758 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-19 17:50:32,758 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-19 17:50:32,759 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-19 17:50:32,759 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-19 17:50:32,760 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-19 17:50:32,760 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-19 17:50:32,760 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-19 17:50:32,760 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-19 17:50:32,761 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-19 17:50:32,763 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-19 17:50:32,764 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@4331d187) with optional ClassLoader: null 2024-11-19 17:50:32,764 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-19 17:50:32,765 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@4331d187] started OK. 2024-11-19T17:50:32,781 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync timeout: 13 mins 2024-11-19 17:50:32,784 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-19 17:50:32,784 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-19T17:50:32,834 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.TestFailedAppendAndSync#testLockupAroundBadAssignSync Thread=11, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=627, ProcessCount=11, AvailableMemoryMB=10187 2024-11-19T17:50:33,097 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673 2024-11-19T17:50:33,678 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-19T17:50:33,792 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-19T17:50:33,850 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=wal, suffix=, logDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync, archiveDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/oldWALs, maxLogs=55 2024-11-19T17:50:33,898 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038633889 2024-11-19T17:50:33,970 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038633889 2024-11-19T17:50:34,042 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:34,051 INFO [Time-limited test {}] regionserver.ChunkCreator(472): data poolSizePercentage is less than 0. So not using pool 2024-11-19T17:50:34,051 INFO [Time-limited test {}] regionserver.ChunkCreator(472): index poolSizePercentage is less than 0. So not using pool 2024-11-19T17:50:34,098 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => c82f2c332c5429d4b3c9bb758165d0bc, NAME => 'testLockupAroundBadAssignSync,,1732038634069.c82f2c332c5429d4b3c9bb758165d0bc.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testLockupAroundBadAssignSync', {TABLE_ATTRIBUTES => {DURABILITY => 'SYNC_WAL', READONLY => 'false'}}, {NAME => 'MyCF', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)', METADATA => {'IN_MEMORY_COMPACTION' => 'NONE'}}, regionDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673 2024-11-19T17:50:34,158 INFO [Time-limited test {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-19T17:50:34,163 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testLockupAroundBadAssignSync,,1732038634069.c82f2c332c5429d4b3c9bb758165d0bc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T17:50:34,254 INFO [StoreOpener-c82f2c332c5429d4b3c9bb758165d0bc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family MyCF of region c82f2c332c5429d4b3c9bb758165d0bc 2024-11-19T17:50:34,323 INFO [StoreOpener-c82f2c332c5429d4b3c9bb758165d0bc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c82f2c332c5429d4b3c9bb758165d0bc columnFamilyName MyCF 2024-11-19T17:50:34,330 DEBUG [StoreOpener-c82f2c332c5429d4b3c9bb758165d0bc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:50:34,337 INFO [StoreOpener-c82f2c332c5429d4b3c9bb758165d0bc-1 {}] regionserver.HStore(327): Store=c82f2c332c5429d4b3c9bb758165d0bc/MyCF, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:50:34,342 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c82f2c332c5429d4b3c9bb758165d0bc 2024-11-19T17:50:34,348 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/data/default/testLockupAroundBadAssignSync/c82f2c332c5429d4b3c9bb758165d0bc 2024-11-19T17:50:34,349 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/data/default/testLockupAroundBadAssignSync/c82f2c332c5429d4b3c9bb758165d0bc 2024-11-19T17:50:34,351 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c82f2c332c5429d4b3c9bb758165d0bc 2024-11-19T17:50:34,352 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c82f2c332c5429d4b3c9bb758165d0bc 2024-11-19T17:50:34,360 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c82f2c332c5429d4b3c9bb758165d0bc 2024-11-19T17:50:34,372 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/data/default/testLockupAroundBadAssignSync/c82f2c332c5429d4b3c9bb758165d0bc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T17:50:34,373 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c82f2c332c5429d4b3c9bb758165d0bc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70099142, jitterRate=0.04455861449241638}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T17:50:34,382 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-19T17:50:34,399 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c82f2c332c5429d4b3c9bb758165d0bc: Writing region info on filesystem at 1732038634216Initializing all the Stores at 1732038634218 (+2 ms)Instantiating store for column family {NAME => 'MyCF', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)', METADATA => {'IN_MEMORY_COMPACTION' => 'NONE'}} at 1732038634218Cleaning up temporary data from old regions at 1732038634353 (+135 ms)Region opened successfully at 1732038634396 (+43 ms) 2024-11-19T17:50:34,439 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,444 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038633889) roll requested 2024-11-19T17:50:34,449 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038634449 2024-11-19T17:50:34,457 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,458 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,458 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,458 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,459 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,459 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038633889 with entries=1, filesize=249 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634449 2024-11-19T17:50:34,468 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:34,468 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,469 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,469 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038634449) roll requested 2024-11-19T17:50:34,470 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038634470 2024-11-19T17:50:34,478 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,478 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,478 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,479 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,479 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634449 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634470 2024-11-19T17:50:34,486 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:34,486 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,487 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038634470) roll requested 2024-11-19T17:50:34,488 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038634488 2024-11-19T17:50:34,494 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,494 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,495 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634470 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634488 2024-11-19T17:50:34,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:34,496 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,498 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038634488) roll requested 2024-11-19T17:50:34,498 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038634498 2024-11-19T17:50:34,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,508 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,508 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,508 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,509 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634488 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634498 2024-11-19T17:50:34,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:34,514 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038634498) roll requested 2024-11-19T17:50:34,516 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038634515 2024-11-19T17:50:34,523 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,524 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,524 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,524 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,524 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634498 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634515 2024-11-19T17:50:34,526 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:34,526 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,526 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038634515) roll requested 2024-11-19T17:50:34,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038634527 2024-11-19T17:50:34,535 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,535 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,536 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,536 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,536 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,537 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634515 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634527 2024-11-19T17:50:34,537 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:34,538 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,538 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,539 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038634527) roll requested 2024-11-19T17:50:34,539 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038634539 2024-11-19T17:50:34,553 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,553 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,553 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,553 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,554 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,554 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634527 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634539 2024-11-19T17:50:34,558 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:34,558 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,559 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,559 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038634539) roll requested 2024-11-19T17:50:34,560 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038634559 2024-11-19T17:50:34,567 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,568 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,568 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,568 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,569 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634539 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634559 2024-11-19T17:50:34,570 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:34,570 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,570 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038634559) roll requested 2024-11-19T17:50:34,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038634571 2024-11-19T17:50:34,579 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,580 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,580 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,580 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,580 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634559 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634571 2024-11-19T17:50:34,583 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:34,583 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,583 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038634571) roll requested 2024-11-19T17:50:34,584 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038634584 2024-11-19T17:50:34,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,593 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,593 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,593 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,593 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,593 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634571 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634584 2024-11-19T17:50:34,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:34,595 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,595 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,596 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038634584) roll requested 2024-11-19T17:50:34,596 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038634596 2024-11-19T17:50:34,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,603 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,603 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,603 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,603 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,604 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634584 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634596 2024-11-19T17:50:34,606 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:34,606 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,606 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,607 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038634596) roll requested 2024-11-19T17:50:34,607 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038634607 2024-11-19T17:50:34,613 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,613 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,613 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,614 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,614 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,614 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634596 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634607 2024-11-19T17:50:34,615 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,615 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,615 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:34,616 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038634607) roll requested 2024-11-19T17:50:34,616 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038634616 2024-11-19T17:50:34,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634607 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634616 2024-11-19T17:50:34,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:34,624 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038634616) roll requested 2024-11-19T17:50:34,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038634625 2024-11-19T17:50:34,631 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,631 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,632 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,632 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634616 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634625 2024-11-19T17:50:34,633 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:34,633 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,633 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,633 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038634625) roll requested 2024-11-19T17:50:34,634 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038634634 2024-11-19T17:50:34,640 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,640 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,640 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,641 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,641 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,641 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634625 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634634 2024-11-19T17:50:34,642 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:34,642 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,642 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,643 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038634634) roll requested 2024-11-19T17:50:34,643 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038634643 2024-11-19T17:50:34,648 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,648 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,649 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,649 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,649 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,649 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634634 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634643 2024-11-19T17:50:34,650 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:34,650 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,650 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038634643) roll requested 2024-11-19T17:50:34,651 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038634651 2024-11-19T17:50:34,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,660 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,660 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634643 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634651 2024-11-19T17:50:34,662 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:34,662 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,662 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,662 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038634651) roll requested 2024-11-19T17:50:34,663 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038634662 2024-11-19T17:50:34,668 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,668 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,669 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,669 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,669 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,669 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634651 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634662 2024-11-19T17:50:34,670 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:34,670 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,670 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,671 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038634662) roll requested 2024-11-19T17:50:34,671 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038634671 2024-11-19T17:50:34,676 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,677 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,677 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,677 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634662 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634671 2024-11-19T17:50:34,678 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:34,678 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,679 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,679 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038634671) roll requested 2024-11-19T17:50:34,680 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038634679 2024-11-19T17:50:34,687 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634671 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634679 2024-11-19T17:50:34,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:34,689 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,690 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038634679) roll requested 2024-11-19T17:50:34,690 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038634690 2024-11-19T17:50:34,697 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,697 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,697 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,697 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,697 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,698 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634679 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634690 2024-11-19T17:50:34,703 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:34,703 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038634690) roll requested 2024-11-19T17:50:34,705 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038634704 2024-11-19T17:50:34,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,711 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,711 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634690 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634704 2024-11-19T17:50:34,718 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:34,718 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,719 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,719 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038634704) roll requested 2024-11-19T17:50:34,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038634719 2024-11-19T17:50:34,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,726 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634704 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634719 2024-11-19T17:50:34,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:34,732 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,732 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038634719) roll requested 2024-11-19T17:50:34,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038634733 2024-11-19T17:50:34,740 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,740 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,740 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,740 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,740 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634719 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634733 2024-11-19T17:50:34,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:34,742 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,742 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038634733) roll requested 2024-11-19T17:50:34,742 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038634742 2024-11-19T17:50:34,748 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,749 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,749 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,749 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,749 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,749 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634733 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634742 2024-11-19T17:50:34,750 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:34,750 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,750 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,750 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038634742) roll requested 2024-11-19T17:50:34,751 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038634750 2024-11-19T17:50:34,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,762 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,762 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,763 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634742 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634750 2024-11-19T17:50:34,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:34,764 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,765 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,765 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038634750) roll requested 2024-11-19T17:50:34,765 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038634765 2024-11-19T17:50:34,770 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,770 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,771 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,771 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634750 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634765 2024-11-19T17:50:34,773 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:34,773 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038634765) roll requested 2024-11-19T17:50:34,774 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038634773 2024-11-19T17:50:34,779 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,779 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,779 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,780 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,780 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,780 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634765 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634773 2024-11-19T17:50:34,781 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:34,781 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,781 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,781 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038634773) roll requested 2024-11-19T17:50:34,782 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038634782 2024-11-19T17:50:34,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,788 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,788 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,788 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,788 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634773 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634782 2024-11-19T17:50:34,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:34,789 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,790 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038634782) roll requested 2024-11-19T17:50:34,790 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038634790 2024-11-19T17:50:34,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634782 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634790 2024-11-19T17:50:34,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:34,799 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,800 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038634790) roll requested 2024-11-19T17:50:34,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038634800 2024-11-19T17:50:34,808 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,808 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,808 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,808 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,809 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,809 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634790 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634800 2024-11-19T17:50:34,810 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:34,810 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,810 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,810 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038634800) roll requested 2024-11-19T17:50:34,811 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038634810 2024-11-19T17:50:34,816 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,817 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,817 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,817 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,817 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,817 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634800 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634810 2024-11-19T17:50:34,818 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:34,818 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,818 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,819 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038634810) roll requested 2024-11-19T17:50:34,819 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038634819 2024-11-19T17:50:34,825 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,825 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,825 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,825 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,825 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,825 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634810 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634819 2024-11-19T17:50:34,826 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:34,826 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,826 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,826 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038634819) roll requested 2024-11-19T17:50:34,827 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038634827 2024-11-19T17:50:34,832 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,832 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,832 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,832 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,832 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,832 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634819 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634827 2024-11-19T17:50:34,833 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:34,833 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,834 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,834 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038634827) roll requested 2024-11-19T17:50:34,834 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038634834 2024-11-19T17:50:34,840 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,841 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,841 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634827 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634834 2024-11-19T17:50:34,843 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:34,843 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,843 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,844 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038634834) roll requested 2024-11-19T17:50:34,844 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038634844 2024-11-19T17:50:34,850 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,850 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,850 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,851 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,851 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,851 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634834 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634844 2024-11-19T17:50:34,852 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:34,852 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,852 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,853 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038634844) roll requested 2024-11-19T17:50:34,853 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038634853 2024-11-19T17:50:34,860 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,860 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,860 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,861 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,861 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,861 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634844 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634853 2024-11-19T17:50:34,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:34,867 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,867 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038634853) roll requested 2024-11-19T17:50:34,868 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038634868 2024-11-19T17:50:34,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,878 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,878 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,878 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634853 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634868 2024-11-19T17:50:34,879 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:34,879 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,879 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,879 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038634868) roll requested 2024-11-19T17:50:34,879 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038634879 2024-11-19T17:50:34,888 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,889 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,889 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634868 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634879 2024-11-19T17:50:34,892 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:34,892 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,892 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038634879) roll requested 2024-11-19T17:50:34,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038634893 2024-11-19T17:50:34,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634879 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634893 2024-11-19T17:50:34,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:34,900 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038634893) roll requested 2024-11-19T17:50:34,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038634901 2024-11-19T17:50:34,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634893 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634901 2024-11-19T17:50:34,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:34,908 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,908 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038634901) roll requested 2024-11-19T17:50:34,908 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038634908 2024-11-19T17:50:34,914 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,914 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,914 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,915 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634901 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634908 2024-11-19T17:50:34,915 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:34,915 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,916 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,916 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038634908) roll requested 2024-11-19T17:50:34,916 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038634916 2024-11-19T17:50:34,922 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,923 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,923 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,923 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,923 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,923 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634908 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634916 2024-11-19T17:50:34,924 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:34,924 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038634916) roll requested 2024-11-19T17:50:34,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038634925 2024-11-19T17:50:34,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634916 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634925 2024-11-19T17:50:34,933 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:34,933 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,933 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038634925) roll requested 2024-11-19T17:50:34,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038634933 2024-11-19T17:50:34,938 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,939 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,939 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,939 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634925 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634933 2024-11-19T17:50:34,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:34,940 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038634933) roll requested 2024-11-19T17:50:34,941 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038634940 2024-11-19T17:50:34,948 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,948 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,949 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,949 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,949 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,949 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634933 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634940 2024-11-19T17:50:34,950 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:34,950 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,950 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,950 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038634940) roll requested 2024-11-19T17:50:34,950 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038634950 2024-11-19T17:50:34,956 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,956 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,956 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,956 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634940 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634950 2024-11-19T17:50:34,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:34,958 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038634950) roll requested 2024-11-19T17:50:34,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038634958 2024-11-19T17:50:34,963 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,963 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,963 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,964 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,964 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,964 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634950 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634958 2024-11-19T17:50:34,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:34,965 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038634958) roll requested 2024-11-19T17:50:34,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038634965 2024-11-19T17:50:34,975 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,975 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,976 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634958 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634965 2024-11-19T17:50:34,977 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:34,977 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,977 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038634965) roll requested 2024-11-19T17:50:34,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038634977 2024-11-19T17:50:34,983 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,984 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,984 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,984 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,984 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,984 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634965 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634977 2024-11-19T17:50:34,985 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:34,985 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,985 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038634977) roll requested 2024-11-19T17:50:34,986 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038634985 2024-11-19T17:50:34,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:34,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634977 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634985 2024-11-19T17:50:34,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:34,996 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,996 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:34,996 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038634985) roll requested 2024-11-19T17:50:34,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038634997 2024-11-19T17:50:35,005 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,006 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,006 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634985 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634997 2024-11-19T17:50:35,007 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,007 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,008 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038634997) roll requested 2024-11-19T17:50:35,008 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635008 2024-11-19T17:50:35,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,017 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038634997 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635008 2024-11-19T17:50:35,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,018 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,018 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,019 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635008) roll requested 2024-11-19T17:50:35,019 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635019 2024-11-19T17:50:35,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,025 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,025 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,025 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,025 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,025 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635008 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635019 2024-11-19T17:50:35,026 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,026 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,026 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,026 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635019) roll requested 2024-11-19T17:50:35,027 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635027 2024-11-19T17:50:35,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,032 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,032 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,032 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635019 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635027 2024-11-19T17:50:35,033 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,033 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,033 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635027) roll requested 2024-11-19T17:50:35,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635033 2024-11-19T17:50:35,039 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,039 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,039 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,039 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,039 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,039 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635027 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635033 2024-11-19T17:50:35,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,040 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635033) roll requested 2024-11-19T17:50:35,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635041 2024-11-19T17:50:35,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635033 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635041 2024-11-19T17:50:35,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,048 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,049 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635041) roll requested 2024-11-19T17:50:35,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635049 2024-11-19T17:50:35,054 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,054 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,055 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,055 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,055 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635041 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635049 2024-11-19T17:50:35,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,056 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635049) roll requested 2024-11-19T17:50:35,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635057 2024-11-19T17:50:35,064 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,064 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,064 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,064 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,065 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,065 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635049 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635057 2024-11-19T17:50:35,066 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,066 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,066 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,066 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635057) roll requested 2024-11-19T17:50:35,067 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635066 2024-11-19T17:50:35,074 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,074 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,074 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,074 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,074 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,074 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635057 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635066 2024-11-19T17:50:35,075 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,075 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,075 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,075 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635066) roll requested 2024-11-19T17:50:35,076 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635076 2024-11-19T17:50:35,083 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,083 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,083 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,083 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,083 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635066 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635076 2024-11-19T17:50:35,084 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,084 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,084 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,084 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635076) roll requested 2024-11-19T17:50:35,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635085 2024-11-19T17:50:35,090 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,091 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,091 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635076 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635085 2024-11-19T17:50:35,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,092 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,092 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635085) roll requested 2024-11-19T17:50:35,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635092 2024-11-19T17:50:35,102 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,102 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,102 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,102 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635085 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635092 2024-11-19T17:50:35,104 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,104 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,104 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635092) roll requested 2024-11-19T17:50:35,104 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635104 2024-11-19T17:50:35,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,120 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,120 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635092 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635104 2024-11-19T17:50:35,121 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,121 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635104) roll requested 2024-11-19T17:50:35,122 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635122 2024-11-19T17:50:35,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635104 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635122 2024-11-19T17:50:35,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,129 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635122) roll requested 2024-11-19T17:50:35,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635129 2024-11-19T17:50:35,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,135 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,135 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,135 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635122 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635129 2024-11-19T17:50:35,136 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,136 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,136 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,136 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635129) roll requested 2024-11-19T17:50:35,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635137 2024-11-19T17:50:35,143 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,143 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,143 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,143 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,143 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,143 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635129 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635137 2024-11-19T17:50:35,144 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,144 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,144 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,144 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635137) roll requested 2024-11-19T17:50:35,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635145 2024-11-19T17:50:35,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,151 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635137 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635145 2024-11-19T17:50:35,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,152 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635145) roll requested 2024-11-19T17:50:35,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635153 2024-11-19T17:50:35,159 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,159 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,159 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,159 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635145 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635153 2024-11-19T17:50:35,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,160 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635153) roll requested 2024-11-19T17:50:35,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635161 2024-11-19T17:50:35,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635153 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635161 2024-11-19T17:50:35,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,168 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,168 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635161) roll requested 2024-11-19T17:50:35,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635168 2024-11-19T17:50:35,174 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,174 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,174 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,174 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,174 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,174 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635161 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635168 2024-11-19T17:50:35,175 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,175 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,175 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,175 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635168) roll requested 2024-11-19T17:50:35,176 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635175 2024-11-19T17:50:35,181 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,181 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,181 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,182 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,182 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635168 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635175 2024-11-19T17:50:35,182 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,182 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,183 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,183 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635175) roll requested 2024-11-19T17:50:35,183 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635183 2024-11-19T17:50:35,190 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,190 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,190 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,190 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,190 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,190 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635175 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635183 2024-11-19T17:50:35,191 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,191 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,191 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,191 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635183) roll requested 2024-11-19T17:50:35,192 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635191 2024-11-19T17:50:35,197 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,197 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,198 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,198 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,198 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,198 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635183 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635191 2024-11-19T17:50:35,199 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,199 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,199 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,199 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635191) roll requested 2024-11-19T17:50:35,199 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635199 2024-11-19T17:50:35,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,205 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,205 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,205 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,205 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635191 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635199 2024-11-19T17:50:35,206 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,206 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,206 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,206 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635199) roll requested 2024-11-19T17:50:35,206 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635206 2024-11-19T17:50:35,212 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,213 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,213 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635199 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635206 2024-11-19T17:50:35,214 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,214 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,214 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,214 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635206) roll requested 2024-11-19T17:50:35,214 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635214 2024-11-19T17:50:35,220 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,220 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,220 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,220 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,220 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,220 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635206 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635214 2024-11-19T17:50:35,222 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,222 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,222 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,222 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635214) roll requested 2024-11-19T17:50:35,222 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635222 2024-11-19T17:50:35,227 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635214 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635222 2024-11-19T17:50:35,228 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,228 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635222) roll requested 2024-11-19T17:50:35,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635229 2024-11-19T17:50:35,235 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,235 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,235 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,235 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,235 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,235 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635222 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635229 2024-11-19T17:50:35,236 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,236 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,236 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,236 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635229) roll requested 2024-11-19T17:50:35,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635236 2024-11-19T17:50:35,243 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,243 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,244 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,244 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,244 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635229 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635236 2024-11-19T17:50:35,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,244 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,245 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635236) roll requested 2024-11-19T17:50:35,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635245 2024-11-19T17:50:35,250 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635236 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635245 2024-11-19T17:50:35,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,251 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,251 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635245) roll requested 2024-11-19T17:50:35,251 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635251 2024-11-19T17:50:35,256 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,256 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,256 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,256 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,257 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,257 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635245 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635251 2024-11-19T17:50:35,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,257 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,258 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635251) roll requested 2024-11-19T17:50:35,258 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635258 2024-11-19T17:50:35,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,263 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,263 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,263 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,263 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,263 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635251 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635258 2024-11-19T17:50:35,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,264 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635258) roll requested 2024-11-19T17:50:35,264 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635264 2024-11-19T17:50:35,270 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,270 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,270 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,270 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,270 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635258 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635264 2024-11-19T17:50:35,271 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,271 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,271 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,272 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635264) roll requested 2024-11-19T17:50:35,272 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635272 2024-11-19T17:50:35,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635264 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635272 2024-11-19T17:50:35,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,279 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,279 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,279 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635272) roll requested 2024-11-19T17:50:35,280 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635280 2024-11-19T17:50:35,287 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,287 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,287 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635272 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635280 2024-11-19T17:50:35,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,288 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,288 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,289 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635280) roll requested 2024-11-19T17:50:35,289 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635289 2024-11-19T17:50:35,295 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,295 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,295 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,295 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635280 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635289 2024-11-19T17:50:35,296 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,296 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,296 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,296 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635289) roll requested 2024-11-19T17:50:35,297 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635297 2024-11-19T17:50:35,302 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,302 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,303 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,303 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,303 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635289 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635297 2024-11-19T17:50:35,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,304 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635297) roll requested 2024-11-19T17:50:35,304 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635304 2024-11-19T17:50:35,309 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,309 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,310 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,310 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,310 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,310 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635297 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635304 2024-11-19T17:50:35,310 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,310 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,311 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,311 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635304) roll requested 2024-11-19T17:50:35,311 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635311 2024-11-19T17:50:35,316 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,316 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635304 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635311 2024-11-19T17:50:35,317 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,317 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635311) roll requested 2024-11-19T17:50:35,318 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635317 2024-11-19T17:50:35,323 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,323 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,323 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,324 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,324 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,324 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635311 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635317 2024-11-19T17:50:35,325 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,325 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,325 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,325 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635317) roll requested 2024-11-19T17:50:35,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635325 2024-11-19T17:50:35,330 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,331 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,331 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,331 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,331 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635317 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635325 2024-11-19T17:50:35,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,332 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,332 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635325) roll requested 2024-11-19T17:50:35,332 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635332 2024-11-19T17:50:35,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,338 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,339 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635325 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635332 2024-11-19T17:50:35,339 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,339 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,340 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635332) roll requested 2024-11-19T17:50:35,340 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635340 2024-11-19T17:50:35,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,346 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,346 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635332 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635340 2024-11-19T17:50:35,349 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,349 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635340) roll requested 2024-11-19T17:50:35,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635350 2024-11-19T17:50:35,355 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,355 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,355 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635340 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635350 2024-11-19T17:50:35,359 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,359 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,359 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,359 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635350) roll requested 2024-11-19T17:50:35,359 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635359 2024-11-19T17:50:35,365 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,365 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,365 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,365 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635350 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635359 2024-11-19T17:50:35,366 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,366 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,366 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,366 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635359) roll requested 2024-11-19T17:50:35,367 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635366 2024-11-19T17:50:35,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,372 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,372 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,372 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,372 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,372 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635359 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635366 2024-11-19T17:50:35,373 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,373 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,373 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,373 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635366) roll requested 2024-11-19T17:50:35,373 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635373 2024-11-19T17:50:35,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,379 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,379 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,379 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,379 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635366 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635373 2024-11-19T17:50:35,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,380 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635373) roll requested 2024-11-19T17:50:35,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635380 2024-11-19T17:50:35,386 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,386 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,386 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,386 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635373 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635380 2024-11-19T17:50:35,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,387 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,388 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635380) roll requested 2024-11-19T17:50:35,388 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635388 2024-11-19T17:50:35,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635380 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635388 2024-11-19T17:50:35,394 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,394 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,394 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635388) roll requested 2024-11-19T17:50:35,395 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635395 2024-11-19T17:50:35,400 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,400 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,401 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,401 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,401 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,401 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635388 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635395 2024-11-19T17:50:35,402 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,402 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,402 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,402 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635395) roll requested 2024-11-19T17:50:35,402 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635402 2024-11-19T17:50:35,408 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,408 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,408 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,408 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,408 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,408 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635395 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635402 2024-11-19T17:50:35,409 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,409 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,409 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,409 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635402) roll requested 2024-11-19T17:50:35,409 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635409 2024-11-19T17:50:35,415 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,415 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,415 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,415 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,415 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,415 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635402 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635409 2024-11-19T17:50:35,416 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,416 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,416 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,416 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635409) roll requested 2024-11-19T17:50:35,417 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635416 2024-11-19T17:50:35,425 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,425 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,426 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,426 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635409 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635416 2024-11-19T17:50:35,426 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,426 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635416) roll requested 2024-11-19T17:50:35,427 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635427 2024-11-19T17:50:35,432 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,432 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,432 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,432 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635416 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635427 2024-11-19T17:50:35,433 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,433 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,434 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635427) roll requested 2024-11-19T17:50:35,434 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635434 2024-11-19T17:50:35,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,440 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635427 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635434 2024-11-19T17:50:35,442 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,442 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,442 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,442 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635434) roll requested 2024-11-19T17:50:35,443 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635442 2024-11-19T17:50:35,448 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,448 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,448 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,448 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,448 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635434 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635442 2024-11-19T17:50:35,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,449 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635442) roll requested 2024-11-19T17:50:35,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635450 2024-11-19T17:50:35,455 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,455 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,455 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,455 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635442 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635450 2024-11-19T17:50:35,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,456 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635450) roll requested 2024-11-19T17:50:35,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635457 2024-11-19T17:50:35,462 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635450 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635457 2024-11-19T17:50:35,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,464 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635457) roll requested 2024-11-19T17:50:35,464 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635464 2024-11-19T17:50:35,469 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,470 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,470 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,470 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635457 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635464 2024-11-19T17:50:35,471 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,471 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,471 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,471 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635464) roll requested 2024-11-19T17:50:35,471 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635471 2024-11-19T17:50:35,476 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,476 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,477 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,477 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635464 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635471 2024-11-19T17:50:35,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,477 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635471) roll requested 2024-11-19T17:50:35,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635478 2024-11-19T17:50:35,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,483 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,484 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,484 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,484 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635471 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635478 2024-11-19T17:50:35,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,484 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,485 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635478) roll requested 2024-11-19T17:50:35,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635485 2024-11-19T17:50:35,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,491 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,491 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,491 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,491 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635478 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635485 2024-11-19T17:50:35,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,492 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635485) roll requested 2024-11-19T17:50:35,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635493 2024-11-19T17:50:35,498 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,498 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,498 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,498 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,498 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635485 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635493 2024-11-19T17:50:35,503 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,503 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635493) roll requested 2024-11-19T17:50:35,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635504 2024-11-19T17:50:35,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635493 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635504 2024-11-19T17:50:35,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,512 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,512 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635504) roll requested 2024-11-19T17:50:35,512 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635512 2024-11-19T17:50:35,518 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,518 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,518 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,518 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,519 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635504 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635512 2024-11-19T17:50:35,519 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,519 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,519 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,520 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635512) roll requested 2024-11-19T17:50:35,520 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635520 2024-11-19T17:50:35,525 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635512 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635520 2024-11-19T17:50:35,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,527 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635520) roll requested 2024-11-19T17:50:35,527 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635527 2024-11-19T17:50:35,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635520 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635527 2024-11-19T17:50:35,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,535 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,535 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635527) roll requested 2024-11-19T17:50:35,535 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635535 2024-11-19T17:50:35,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,542 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635527 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635535 2024-11-19T17:50:35,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,544 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,544 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635535) roll requested 2024-11-19T17:50:35,544 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635544 2024-11-19T17:50:35,550 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,550 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,550 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,550 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,550 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,551 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635535 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635544 2024-11-19T17:50:35,551 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,551 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,551 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,551 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635544) roll requested 2024-11-19T17:50:35,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635552 2024-11-19T17:50:35,557 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,557 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,557 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,557 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,557 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,557 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635544 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635552 2024-11-19T17:50:35,558 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,558 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,558 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,558 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635552) roll requested 2024-11-19T17:50:35,559 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635559 2024-11-19T17:50:35,564 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,564 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,564 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,564 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,565 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,565 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635552 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635559 2024-11-19T17:50:35,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,565 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,566 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635559) roll requested 2024-11-19T17:50:35,566 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635566 2024-11-19T17:50:35,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,572 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,572 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,572 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,572 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635559 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635566 2024-11-19T17:50:35,573 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,573 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,574 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,574 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635566) roll requested 2024-11-19T17:50:35,574 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635574 2024-11-19T17:50:35,579 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,579 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,580 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635566 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635574 2024-11-19T17:50:35,581 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,581 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,581 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635574) roll requested 2024-11-19T17:50:35,581 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635581 2024-11-19T17:50:35,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,603 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,603 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,603 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635574 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635581 2024-11-19T17:50:35,603 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,603 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635574 is not closed yet, will try archiving it next time 2024-11-19T17:50:35,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635581) roll requested 2024-11-19T17:50:35,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635604 2024-11-19T17:50:35,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,619 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,619 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,619 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,619 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,619 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635581 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635604 2024-11-19T17:50:35,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,620 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,620 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635604) roll requested 2024-11-19T17:50:35,620 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635620 2024-11-19T17:50:35,632 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,632 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,632 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,632 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,632 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,632 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635604 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635620 2024-11-19T17:50:35,636 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,636 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,636 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,636 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635620) roll requested 2024-11-19T17:50:35,637 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635636 2024-11-19T17:50:35,641 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,642 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,642 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,642 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,642 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635620 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635636 2024-11-19T17:50:35,642 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,642 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,643 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,643 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635636) roll requested 2024-11-19T17:50:35,643 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635643 2024-11-19T17:50:35,649 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,649 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,649 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,650 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,650 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,650 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635636 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635643 2024-11-19T17:50:35,650 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,650 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,651 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635643) roll requested 2024-11-19T17:50:35,651 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635651 2024-11-19T17:50:35,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,657 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,657 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635643 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635651 2024-11-19T17:50:35,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,664 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635651) roll requested 2024-11-19T17:50:35,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635664 2024-11-19T17:50:35,670 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,670 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,670 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,670 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,670 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,670 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635651 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635664 2024-11-19T17:50:35,671 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,671 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,671 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,671 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635664) roll requested 2024-11-19T17:50:35,671 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635671 2024-11-19T17:50:35,676 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,676 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,676 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,676 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,677 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635664 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635671 2024-11-19T17:50:35,677 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,677 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,678 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,678 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635671) roll requested 2024-11-19T17:50:35,678 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635678 2024-11-19T17:50:35,683 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,683 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,683 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,683 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,683 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,683 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635671 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635678 2024-11-19T17:50:35,685 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,685 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,685 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,685 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635678) roll requested 2024-11-19T17:50:35,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635685 2024-11-19T17:50:35,690 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,691 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,691 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,691 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,691 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,691 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635678 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635685 2024-11-19T17:50:35,692 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,692 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,692 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,692 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635685) roll requested 2024-11-19T17:50:35,693 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635692 2024-11-19T17:50:35,697 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,697 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,697 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,698 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,698 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,698 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635685 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635692 2024-11-19T17:50:35,698 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,698 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,698 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,698 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635692) roll requested 2024-11-19T17:50:35,699 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635699 2024-11-19T17:50:35,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,704 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,704 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,704 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,704 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,704 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635692 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635699 2024-11-19T17:50:35,705 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,705 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,705 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635699) roll requested 2024-11-19T17:50:35,705 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635705 2024-11-19T17:50:35,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,710 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635699 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635705 2024-11-19T17:50:35,711 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,711 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,711 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,711 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635705) roll requested 2024-11-19T17:50:35,712 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635712 2024-11-19T17:50:35,717 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,717 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,718 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,718 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,718 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,718 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635705 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635712 2024-11-19T17:50:35,719 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,719 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,719 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,719 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635712) roll requested 2024-11-19T17:50:35,719 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635719 2024-11-19T17:50:35,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,725 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635712 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635719 2024-11-19T17:50:35,725 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,725 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,725 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635719) roll requested 2024-11-19T17:50:35,726 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635726 2024-11-19T17:50:35,730 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,730 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635719 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635726 2024-11-19T17:50:35,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,732 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,732 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,732 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635726) roll requested 2024-11-19T17:50:35,732 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635732 2024-11-19T17:50:35,737 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,737 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,737 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,737 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,738 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,738 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635726 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635732 2024-11-19T17:50:35,738 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,738 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,738 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,738 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635732) roll requested 2024-11-19T17:50:35,739 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635739 2024-11-19T17:50:35,746 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,746 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,746 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,747 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,747 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,747 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635732 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635739 2024-11-19T17:50:35,747 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,747 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,748 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,748 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635739) roll requested 2024-11-19T17:50:35,748 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635748 2024-11-19T17:50:35,753 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,753 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,753 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,753 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,754 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,754 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635739 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635748 2024-11-19T17:50:35,754 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,754 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,754 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,755 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635748) roll requested 2024-11-19T17:50:35,755 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635755 2024-11-19T17:50:35,761 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,761 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,761 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,761 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,761 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,761 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635748 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635755 2024-11-19T17:50:35,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,764 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635755) roll requested 2024-11-19T17:50:35,765 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635765 2024-11-19T17:50:35,771 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,771 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,771 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,771 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635755 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635765 2024-11-19T17:50:35,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,772 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635765) roll requested 2024-11-19T17:50:35,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635772 2024-11-19T17:50:35,779 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,779 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,779 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,779 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,779 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,779 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635765 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635772 2024-11-19T17:50:35,780 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,780 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,780 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,780 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635772) roll requested 2024-11-19T17:50:35,780 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635780 2024-11-19T17:50:35,785 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,786 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,786 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,786 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,786 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,786 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635772 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635780 2024-11-19T17:50:35,787 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,787 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,787 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,787 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635780) roll requested 2024-11-19T17:50:35,787 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635787 2024-11-19T17:50:35,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,792 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,793 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,793 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,793 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635780 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635787 2024-11-19T17:50:35,793 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,793 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,794 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,794 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635787) roll requested 2024-11-19T17:50:35,794 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635794 2024-11-19T17:50:35,802 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,803 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,803 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,803 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,803 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635787 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635794 2024-11-19T17:50:35,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,804 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635794) roll requested 2024-11-19T17:50:35,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635805 2024-11-19T17:50:35,815 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,815 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,815 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,815 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,816 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,816 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635794 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635805 2024-11-19T17:50:35,817 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,817 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,817 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,817 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635805) roll requested 2024-11-19T17:50:35,817 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635817 2024-11-19T17:50:35,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635805 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635817 2024-11-19T17:50:35,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,836 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635817) roll requested 2024-11-19T17:50:35,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635837 2024-11-19T17:50:35,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,849 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,849 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,849 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,849 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,849 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635817 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635837 2024-11-19T17:50:35,850 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,850 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,850 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,850 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635837) roll requested 2024-11-19T17:50:35,851 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635850 2024-11-19T17:50:35,858 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,858 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,858 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,858 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,858 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,858 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635837 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635850 2024-11-19T17:50:35,859 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,859 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,859 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,859 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635850) roll requested 2024-11-19T17:50:35,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635859 2024-11-19T17:50:35,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635850 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635859 2024-11-19T17:50:35,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,868 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,869 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635859) roll requested 2024-11-19T17:50:35,869 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635869 2024-11-19T17:50:35,875 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,875 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,876 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,876 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,876 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635859 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635869 2024-11-19T17:50:35,877 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,877 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,877 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635869) roll requested 2024-11-19T17:50:35,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635878 2024-11-19T17:50:35,884 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,884 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,884 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,884 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,884 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,885 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635869 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635878 2024-11-19T17:50:35,885 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,885 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,885 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,885 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635878) roll requested 2024-11-19T17:50:35,886 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635886 2024-11-19T17:50:35,891 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,891 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,892 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635878 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635886 2024-11-19T17:50:35,892 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,892 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635886) roll requested 2024-11-19T17:50:35,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635893 2024-11-19T17:50:35,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635886 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635893 2024-11-19T17:50:35,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,901 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635893) roll requested 2024-11-19T17:50:35,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635901 2024-11-19T17:50:35,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,908 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,908 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,908 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,908 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635893 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635901 2024-11-19T17:50:35,909 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,909 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,909 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635901) roll requested 2024-11-19T17:50:35,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635909 2024-11-19T17:50:35,915 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,915 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,915 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,915 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,915 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,915 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635901 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635909 2024-11-19T17:50:35,916 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,916 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,916 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,916 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635909) roll requested 2024-11-19T17:50:35,917 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635916 2024-11-19T17:50:35,923 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,923 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,923 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,923 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,923 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,923 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635909 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635916 2024-11-19T17:50:35,928 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,928 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,928 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,929 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635916) roll requested 2024-11-19T17:50:35,929 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635929 2024-11-19T17:50:35,935 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,935 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,935 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,935 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,936 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,936 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635916 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635929 2024-11-19T17:50:35,936 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,936 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,937 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635929) roll requested 2024-11-19T17:50:35,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635937 2024-11-19T17:50:35,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,944 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,944 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635929 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635937 2024-11-19T17:50:35,949 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,949 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,949 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635937) roll requested 2024-11-19T17:50:35,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635949 2024-11-19T17:50:35,959 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,959 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,960 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,960 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,960 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,960 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635937 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635949 2024-11-19T17:50:35,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,964 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635949) roll requested 2024-11-19T17:50:35,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635965 2024-11-19T17:50:35,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,972 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,972 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,972 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,972 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,972 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635949 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635965 2024-11-19T17:50:35,973 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,973 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,973 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,973 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635965) roll requested 2024-11-19T17:50:35,974 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635973 2024-11-19T17:50:35,983 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,983 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,983 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,983 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,984 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635965 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635973 2024-11-19T17:50:35,984 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,984 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,984 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,984 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635973) roll requested 2024-11-19T17:50:35,985 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635985 2024-11-19T17:50:35,997 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,997 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,997 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:35,998 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635973 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635985 2024-11-19T17:50:35,998 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:35,998 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,999 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:35,999 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635985) roll requested 2024-11-19T17:50:35,999 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038635999 2024-11-19T17:50:36,009 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,009 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635985 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635999 2024-11-19T17:50:36,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,012 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,013 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,013 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038635999) roll requested 2024-11-19T17:50:36,013 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636013 2024-11-19T17:50:36,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,019 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,019 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,020 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,020 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,020 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038635999 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636013 2024-11-19T17:50:36,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,020 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,021 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,021 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636013) roll requested 2024-11-19T17:50:36,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636021 2024-11-19T17:50:36,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,032 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,032 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,032 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636013 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636021 2024-11-19T17:50:36,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,032 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636021) roll requested 2024-11-19T17:50:36,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636033 2024-11-19T17:50:36,043 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,043 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,043 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,043 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,043 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,043 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636021 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636033 2024-11-19T17:50:36,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,044 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,044 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636033) roll requested 2024-11-19T17:50:36,044 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636044 2024-11-19T17:50:36,052 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,053 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,053 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,053 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,053 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,053 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636033 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636044 2024-11-19T17:50:36,054 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,054 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,054 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,054 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636044) roll requested 2024-11-19T17:50:36,054 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636054 2024-11-19T17:50:36,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,061 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636044 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636054 2024-11-19T17:50:36,062 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,062 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,062 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,062 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636054) roll requested 2024-11-19T17:50:36,062 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636062 2024-11-19T17:50:36,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,070 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,070 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,070 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,070 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636054 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636062 2024-11-19T17:50:36,071 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,071 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,071 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,071 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636062) roll requested 2024-11-19T17:50:36,071 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636071 2024-11-19T17:50:36,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,082 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,082 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,082 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,082 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636062 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636071 2024-11-19T17:50:36,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,083 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,083 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636071) roll requested 2024-11-19T17:50:36,084 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636083 2024-11-19T17:50:36,090 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,090 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,090 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,090 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,090 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,090 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636071 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636083 2024-11-19T17:50:36,091 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,091 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,091 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,091 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636083) roll requested 2024-11-19T17:50:36,092 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636092 2024-11-19T17:50:36,098 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,098 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,098 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,098 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,099 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636083 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636092 2024-11-19T17:50:36,100 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,100 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,100 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,100 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636092) roll requested 2024-11-19T17:50:36,101 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636101 2024-11-19T17:50:36,107 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,107 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,107 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,108 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,108 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,108 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636092 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636101 2024-11-19T17:50:36,109 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,109 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,109 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,109 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636101) roll requested 2024-11-19T17:50:36,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636109 2024-11-19T17:50:36,116 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,116 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,116 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,116 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,116 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,116 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636101 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636109 2024-11-19T17:50:36,117 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,117 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,117 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,117 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636109) roll requested 2024-11-19T17:50:36,118 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636117 2024-11-19T17:50:36,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,127 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,127 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,127 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636109 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636117 2024-11-19T17:50:36,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,128 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636117) roll requested 2024-11-19T17:50:36,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636129 2024-11-19T17:50:36,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,135 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,135 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,135 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636117 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636129 2024-11-19T17:50:36,136 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,136 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,137 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,137 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636129) roll requested 2024-11-19T17:50:36,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636137 2024-11-19T17:50:36,143 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,143 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,144 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,144 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,144 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,144 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636129 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636137 2024-11-19T17:50:36,145 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,145 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636137) roll requested 2024-11-19T17:50:36,146 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636145 2024-11-19T17:50:36,155 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,156 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,156 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,156 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,156 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,156 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636137 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636145 2024-11-19T17:50:36,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,160 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636145) roll requested 2024-11-19T17:50:36,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636160 2024-11-19T17:50:36,169 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,169 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,169 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,169 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,169 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,169 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636145 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636160 2024-11-19T17:50:36,170 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,170 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,170 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,170 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636160) roll requested 2024-11-19T17:50:36,171 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636171 2024-11-19T17:50:36,178 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,178 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,178 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,178 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,178 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,178 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636160 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636171 2024-11-19T17:50:36,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,180 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,181 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,181 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636171) roll requested 2024-11-19T17:50:36,181 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636181 2024-11-19T17:50:36,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,188 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636171 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636181 2024-11-19T17:50:36,189 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,189 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636181) roll requested 2024-11-19T17:50:36,190 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636189 2024-11-19T17:50:36,199 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,200 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,200 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,200 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,200 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,200 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636181 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636189 2024-11-19T17:50:36,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,201 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636189) roll requested 2024-11-19T17:50:36,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636201 2024-11-19T17:50:36,211 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,211 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,211 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,211 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,211 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,212 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636189 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636201 2024-11-19T17:50:36,212 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,212 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,213 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,213 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636201) roll requested 2024-11-19T17:50:36,213 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636213 2024-11-19T17:50:36,222 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,222 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,223 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,223 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,223 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636201 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636213 2024-11-19T17:50:36,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,224 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,224 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,224 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636213) roll requested 2024-11-19T17:50:36,224 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636224 2024-11-19T17:50:36,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,230 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,231 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,231 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,231 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636213 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636224 2024-11-19T17:50:36,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,233 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636224) roll requested 2024-11-19T17:50:36,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636233 2024-11-19T17:50:36,242 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,242 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,242 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,242 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,242 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,242 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636224 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636233 2024-11-19T17:50:36,243 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,243 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,243 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,243 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636233) roll requested 2024-11-19T17:50:36,244 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636244 2024-11-19T17:50:36,251 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,251 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,251 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,252 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636233 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636244 2024-11-19T17:50:36,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,257 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,258 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,258 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636244) roll requested 2024-11-19T17:50:36,258 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636258 2024-11-19T17:50:36,264 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,264 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,265 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,265 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636244 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636258 2024-11-19T17:50:36,265 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,265 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,266 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,266 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636258) roll requested 2024-11-19T17:50:36,266 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636266 2024-11-19T17:50:36,271 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,271 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,271 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,271 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636258 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636266 2024-11-19T17:50:36,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,272 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,272 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636266) roll requested 2024-11-19T17:50:36,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636272 2024-11-19T17:50:36,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,279 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636266 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636272 2024-11-19T17:50:36,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,279 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,279 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,279 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636272) roll requested 2024-11-19T17:50:36,280 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636280 2024-11-19T17:50:36,285 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,285 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,286 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,286 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636272 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636280 2024-11-19T17:50:36,286 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,286 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,287 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636280) roll requested 2024-11-19T17:50:36,287 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636287 2024-11-19T17:50:36,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,293 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,293 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636280 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636287 2024-11-19T17:50:36,296 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,296 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,297 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,297 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636287) roll requested 2024-11-19T17:50:36,297 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636297 2024-11-19T17:50:36,302 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,302 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,303 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,303 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,303 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636287 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636297 2024-11-19T17:50:36,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,304 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636297) roll requested 2024-11-19T17:50:36,304 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636304 2024-11-19T17:50:36,310 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,310 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,310 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,310 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,310 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,310 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636297 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636304 2024-11-19T17:50:36,311 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,311 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,311 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,311 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636304) roll requested 2024-11-19T17:50:36,311 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636311 2024-11-19T17:50:36,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,317 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,317 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,317 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,317 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636304 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636311 2024-11-19T17:50:36,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,318 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,318 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,318 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636311) roll requested 2024-11-19T17:50:36,319 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636318 2024-11-19T17:50:36,329 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,329 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,329 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,329 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,329 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,329 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636311 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636318 2024-11-19T17:50:36,330 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,330 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,330 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,330 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636318) roll requested 2024-11-19T17:50:36,331 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636330 2024-11-19T17:50:36,336 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,336 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,336 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,336 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,336 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,336 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636318 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636330 2024-11-19T17:50:36,337 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,337 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,337 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,337 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636330) roll requested 2024-11-19T17:50:36,337 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636337 2024-11-19T17:50:36,343 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636330 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636337 2024-11-19T17:50:36,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,344 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,344 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636337) roll requested 2024-11-19T17:50:36,345 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636344 2024-11-19T17:50:36,350 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,351 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,351 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,351 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,351 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,351 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636337 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636344 2024-11-19T17:50:36,351 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,351 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,352 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,352 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636344) roll requested 2024-11-19T17:50:36,352 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636352 2024-11-19T17:50:36,358 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,358 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,358 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,358 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,358 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,358 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636344 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636352 2024-11-19T17:50:36,359 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,359 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,359 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,359 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636352) roll requested 2024-11-19T17:50:36,359 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636359 2024-11-19T17:50:36,365 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,365 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,365 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,365 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636352 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636359 2024-11-19T17:50:36,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,369 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636359) roll requested 2024-11-19T17:50:36,369 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636369 2024-11-19T17:50:36,374 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,375 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,375 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,375 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,375 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,375 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636359 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636369 2024-11-19T17:50:36,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,379 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,379 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,379 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636369) roll requested 2024-11-19T17:50:36,379 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636379 2024-11-19T17:50:36,384 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,384 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,384 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,384 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,385 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636369 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636379 2024-11-19T17:50:36,386 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,386 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,386 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,386 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636379) roll requested 2024-11-19T17:50:36,387 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636386 2024-11-19T17:50:36,392 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,392 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,392 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636379 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636386 2024-11-19T17:50:36,398 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,398 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,398 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,398 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636386) roll requested 2024-11-19T17:50:36,398 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636398 2024-11-19T17:50:36,404 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,404 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,404 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,405 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636386 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636398 2024-11-19T17:50:36,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,405 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636398) roll requested 2024-11-19T17:50:36,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636406 2024-11-19T17:50:36,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,412 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,412 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,412 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636398 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636406 2024-11-19T17:50:36,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,413 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636406) roll requested 2024-11-19T17:50:36,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636413 2024-11-19T17:50:36,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636406 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636413 2024-11-19T17:50:36,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,420 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636413) roll requested 2024-11-19T17:50:36,420 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636420 2024-11-19T17:50:36,428 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,428 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,428 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,429 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,429 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636413 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636420 2024-11-19T17:50:36,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,432 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636420) roll requested 2024-11-19T17:50:36,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636433 2024-11-19T17:50:36,439 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,439 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636420 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636433 2024-11-19T17:50:36,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,440 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636433) roll requested 2024-11-19T17:50:36,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636440 2024-11-19T17:50:36,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,446 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,447 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,447 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636433 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636440 2024-11-19T17:50:36,447 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,447 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,447 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,448 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636440) roll requested 2024-11-19T17:50:36,448 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636448 2024-11-19T17:50:36,457 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,457 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,457 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,457 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,457 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,457 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636440 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636448 2024-11-19T17:50:36,458 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,458 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,458 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,458 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636448) roll requested 2024-11-19T17:50:36,458 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636458 2024-11-19T17:50:36,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636448 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636458 2024-11-19T17:50:36,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,464 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636458) roll requested 2024-11-19T17:50:36,464 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636464 2024-11-19T17:50:36,469 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,470 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,470 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,470 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636458 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636464 2024-11-19T17:50:36,470 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,470 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,470 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,471 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636464) roll requested 2024-11-19T17:50:36,471 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636471 2024-11-19T17:50:36,476 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,476 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,476 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,477 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,477 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636464 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636471 2024-11-19T17:50:36,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,477 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,477 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636471) roll requested 2024-11-19T17:50:36,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636478 2024-11-19T17:50:36,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,483 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,483 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636471 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636478 2024-11-19T17:50:36,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,484 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636478) roll requested 2024-11-19T17:50:36,484 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636484 2024-11-19T17:50:36,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,490 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,490 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636478 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636484 2024-11-19T17:50:36,491 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,491 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,491 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,491 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636484) roll requested 2024-11-19T17:50:36,491 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636491 2024-11-19T17:50:36,497 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,497 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,497 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,497 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,497 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,497 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636484 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636491 2024-11-19T17:50:36,498 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,498 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,498 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,498 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636491) roll requested 2024-11-19T17:50:36,498 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636498 2024-11-19T17:50:36,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,504 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,504 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,504 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,504 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636491 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636498 2024-11-19T17:50:36,505 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,505 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,505 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636498) roll requested 2024-11-19T17:50:36,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636505 2024-11-19T17:50:36,510 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636498 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636505 2024-11-19T17:50:36,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,512 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,512 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636505) roll requested 2024-11-19T17:50:36,512 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636512 2024-11-19T17:50:36,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,518 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,518 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636505 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636512 2024-11-19T17:50:36,519 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,519 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,519 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,519 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636512) roll requested 2024-11-19T17:50:36,519 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636519 2024-11-19T17:50:36,524 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,524 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,524 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,524 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,524 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,524 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636512 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636519 2024-11-19T17:50:36,525 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,525 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,525 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,525 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636519) roll requested 2024-11-19T17:50:36,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636525 2024-11-19T17:50:36,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,530 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,530 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,530 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636519 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636525 2024-11-19T17:50:36,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,531 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636525) roll requested 2024-11-19T17:50:36,532 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636531 2024-11-19T17:50:36,537 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,537 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,537 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,537 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,537 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,537 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636525 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636531 2024-11-19T17:50:36,538 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,538 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,538 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,538 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636531) roll requested 2024-11-19T17:50:36,538 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636538 2024-11-19T17:50:36,544 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,544 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,544 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,544 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,545 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,545 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636531 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636538 2024-11-19T17:50:36,545 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,545 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,546 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,546 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636538) roll requested 2024-11-19T17:50:36,546 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636546 2024-11-19T17:50:36,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,552 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,552 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,552 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,552 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,552 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636538 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636546 2024-11-19T17:50:36,553 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,553 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,553 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636546) roll requested 2024-11-19T17:50:36,553 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636553 2024-11-19T17:50:36,561 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,561 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,561 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,561 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,561 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,561 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636546 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636553 2024-11-19T17:50:36,562 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,562 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,562 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,562 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636553) roll requested 2024-11-19T17:50:36,563 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636563 2024-11-19T17:50:36,568 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,568 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,568 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,568 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,568 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636553 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636563 2024-11-19T17:50:36,569 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,569 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,569 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,569 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636563) roll requested 2024-11-19T17:50:36,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636569 2024-11-19T17:50:36,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636563 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636569 2024-11-19T17:50:36,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,576 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636569) roll requested 2024-11-19T17:50:36,576 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636576 2024-11-19T17:50:36,585 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,585 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,585 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,585 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,585 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,585 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636569 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636576 2024-11-19T17:50:36,586 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,586 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,586 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,586 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636576) roll requested 2024-11-19T17:50:36,586 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636586 2024-11-19T17:50:36,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,592 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636576 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636586 2024-11-19T17:50:36,596 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,596 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,596 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,596 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636586) roll requested 2024-11-19T17:50:36,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636596 2024-11-19T17:50:36,603 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,603 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,603 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,603 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,603 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,603 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636586 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636596 2024-11-19T17:50:36,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,604 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636596) roll requested 2024-11-19T17:50:36,605 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636604 2024-11-19T17:50:36,610 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,610 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,610 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,610 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,611 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,611 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636596 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636604 2024-11-19T17:50:36,613 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,613 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,613 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,613 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636604) roll requested 2024-11-19T17:50:36,613 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636613 2024-11-19T17:50:36,619 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,620 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,620 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,620 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,620 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,620 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636604 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636613 2024-11-19T17:50:36,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,624 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636613) roll requested 2024-11-19T17:50:36,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636625 2024-11-19T17:50:36,634 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,635 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,635 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636613 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636625 2024-11-19T17:50:36,635 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,635 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,636 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,636 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636625) roll requested 2024-11-19T17:50:36,636 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636636 2024-11-19T17:50:36,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,642 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,642 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,642 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,642 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636625 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636636 2024-11-19T17:50:36,643 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,643 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,643 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,643 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636636) roll requested 2024-11-19T17:50:36,643 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636643 2024-11-19T17:50:36,648 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,649 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,649 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,649 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,649 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,649 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636636 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636643 2024-11-19T17:50:36,650 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,649 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,650 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,650 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636643) roll requested 2024-11-19T17:50:36,650 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636650 2024-11-19T17:50:36,655 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,655 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,655 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,655 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636643 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636650 2024-11-19T17:50:36,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,656 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636650) roll requested 2024-11-19T17:50:36,656 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636656 2024-11-19T17:50:36,668 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,668 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,668 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,668 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,668 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,668 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636650 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636656 2024-11-19T17:50:36,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,669 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,669 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636656) roll requested 2024-11-19T17:50:36,669 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636669 2024-11-19T17:50:36,681 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,681 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636656 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636669 2024-11-19T17:50:36,682 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,682 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,683 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,683 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636669) roll requested 2024-11-19T17:50:36,683 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636683 2024-11-19T17:50:36,689 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,689 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,689 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,689 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,689 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636669 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636683 2024-11-19T17:50:36,690 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,690 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,690 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,690 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636683) roll requested 2024-11-19T17:50:36,690 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636690 2024-11-19T17:50:36,699 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,699 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,699 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,699 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,699 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636683 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636690 2024-11-19T17:50:36,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,700 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636690) roll requested 2024-11-19T17:50:36,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636701 2024-11-19T17:50:36,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636690 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636701 2024-11-19T17:50:36,710 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,710 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,710 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,711 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636701) roll requested 2024-11-19T17:50:36,711 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636711 2024-11-19T17:50:36,716 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,717 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,717 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,717 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,717 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,717 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636701 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636711 2024-11-19T17:50:36,718 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,718 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,718 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,718 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636711) roll requested 2024-11-19T17:50:36,718 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636718 2024-11-19T17:50:36,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,725 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,725 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636711 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636718 2024-11-19T17:50:36,725 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,725 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636718) roll requested 2024-11-19T17:50:36,726 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636726 2024-11-19T17:50:36,734 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,734 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636718 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636726 2024-11-19T17:50:36,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,736 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,736 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636726) roll requested 2024-11-19T17:50:36,737 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636736 2024-11-19T17:50:36,743 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,743 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,743 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,743 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,743 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,743 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636726 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636736 2024-11-19T17:50:36,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,744 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636736) roll requested 2024-11-19T17:50:36,744 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636744 2024-11-19T17:50:36,749 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,749 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,749 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,749 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,749 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636736 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636744 2024-11-19T17:50:36,750 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,750 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,750 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,750 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636744) roll requested 2024-11-19T17:50:36,751 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636750 2024-11-19T17:50:36,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636744 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636750 2024-11-19T17:50:36,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,759 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636750) roll requested 2024-11-19T17:50:36,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636759 2024-11-19T17:50:36,765 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,765 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,765 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,765 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,766 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636750 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636759 2024-11-19T17:50:36,766 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,766 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,766 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636759) roll requested 2024-11-19T17:50:36,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636767 2024-11-19T17:50:36,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636759 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636767 2024-11-19T17:50:36,783 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,783 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,783 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,783 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636767) roll requested 2024-11-19T17:50:36,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636783 2024-11-19T17:50:36,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,791 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,791 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,791 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,791 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636767 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636783 2024-11-19T17:50:36,795 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,795 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,795 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,795 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636783) roll requested 2024-11-19T17:50:36,795 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636795 2024-11-19T17:50:36,806 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,806 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,806 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,806 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,807 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,807 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636783 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636795 2024-11-19T17:50:36,807 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,807 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,807 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,807 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636795) roll requested 2024-11-19T17:50:36,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636808 2024-11-19T17:50:36,816 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,817 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,817 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,817 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,817 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,817 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636795 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636808 2024-11-19T17:50:36,818 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,818 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,818 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,818 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636808) roll requested 2024-11-19T17:50:36,818 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636818 2024-11-19T17:50:36,823 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,823 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,823 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,823 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,823 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,823 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636808 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636818 2024-11-19T17:50:36,824 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,824 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,824 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,824 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636818) roll requested 2024-11-19T17:50:36,824 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636824 2024-11-19T17:50:36,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636818 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636824 2024-11-19T17:50:36,830 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,830 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,830 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,830 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636824) roll requested 2024-11-19T17:50:36,830 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636830 2024-11-19T17:50:36,835 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,835 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,835 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,835 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,835 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,835 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636824 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636830 2024-11-19T17:50:36,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,836 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,836 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636830) roll requested 2024-11-19T17:50:36,836 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636836 2024-11-19T17:50:36,841 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,841 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,841 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,841 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,841 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,841 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636830 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636836 2024-11-19T17:50:36,842 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,842 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,842 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,842 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636836) roll requested 2024-11-19T17:50:36,842 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636842 2024-11-19T17:50:36,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636836 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636842 2024-11-19T17:50:36,851 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,851 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,851 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,851 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636842) roll requested 2024-11-19T17:50:36,851 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636851 2024-11-19T17:50:36,856 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,856 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,857 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,857 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,857 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636842 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636851 2024-11-19T17:50:36,857 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,858 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,858 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,858 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636851) roll requested 2024-11-19T17:50:36,858 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636858 2024-11-19T17:50:36,864 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,864 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,864 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636851 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636858 2024-11-19T17:50:36,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,865 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,865 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636858) roll requested 2024-11-19T17:50:36,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636865 2024-11-19T17:50:36,870 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,870 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,870 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,870 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,870 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,870 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636858 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636865 2024-11-19T17:50:36,871 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,871 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,871 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,871 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636865) roll requested 2024-11-19T17:50:36,872 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636871 2024-11-19T17:50:36,876 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636865 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636871 2024-11-19T17:50:36,877 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,877 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636871) roll requested 2024-11-19T17:50:36,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636878 2024-11-19T17:50:36,883 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,883 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,884 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,884 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,884 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,884 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636871 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636878 2024-11-19T17:50:36,885 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,885 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,885 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,885 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636878) roll requested 2024-11-19T17:50:36,885 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636885 2024-11-19T17:50:36,890 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,890 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,890 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,890 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,890 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,890 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636878 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636885 2024-11-19T17:50:36,891 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,891 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,891 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,891 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636885) roll requested 2024-11-19T17:50:36,891 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636891 2024-11-19T17:50:36,897 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,897 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,897 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,897 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,897 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,897 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636885 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636891 2024-11-19T17:50:36,898 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,898 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,898 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,898 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636891) roll requested 2024-11-19T17:50:36,898 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636898 2024-11-19T17:50:36,903 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,903 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,903 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,903 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,903 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,903 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636891 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636898 2024-11-19T17:50:36,904 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,904 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,904 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,904 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636898) roll requested 2024-11-19T17:50:36,904 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636904 2024-11-19T17:50:36,909 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,910 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,910 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,910 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,910 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636898 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636904 2024-11-19T17:50:36,910 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,910 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,911 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,911 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636904) roll requested 2024-11-19T17:50:36,911 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636911 2024-11-19T17:50:36,915 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,916 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,916 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,916 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,916 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,916 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636904 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636911 2024-11-19T17:50:36,916 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,916 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636911) roll requested 2024-11-19T17:50:36,917 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636917 2024-11-19T17:50:36,922 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,922 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,922 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,923 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,923 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,923 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636911 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636917 2024-11-19T17:50:36,923 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,923 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,923 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,923 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636917) roll requested 2024-11-19T17:50:36,924 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636923 2024-11-19T17:50:36,928 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,929 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,929 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,929 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,929 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,929 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636917 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636923 2024-11-19T17:50:36,930 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,930 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,930 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,930 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636923) roll requested 2024-11-19T17:50:36,930 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636930 2024-11-19T17:50:36,935 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,935 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,935 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,935 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,935 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,935 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636923 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636930 2024-11-19T17:50:36,936 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,936 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,936 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,936 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636930) roll requested 2024-11-19T17:50:36,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636936 2024-11-19T17:50:36,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,945 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636930 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636936 2024-11-19T17:50:36,946 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,946 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,946 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,946 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636936) roll requested 2024-11-19T17:50:36,946 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636946 2024-11-19T17:50:36,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,952 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,952 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,952 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,952 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636936 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636946 2024-11-19T17:50:36,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,952 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,953 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636946) roll requested 2024-11-19T17:50:36,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636953 2024-11-19T17:50:36,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,958 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,958 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,958 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636946 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636953 2024-11-19T17:50:36,959 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,959 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,959 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,959 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636953) roll requested 2024-11-19T17:50:36,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636959 2024-11-19T17:50:36,964 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,964 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,965 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,965 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,965 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636953 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636959 2024-11-19T17:50:36,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,965 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,966 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636959) roll requested 2024-11-19T17:50:36,966 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636966 2024-11-19T17:50:36,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,971 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,971 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,971 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636959 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636966 2024-11-19T17:50:36,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,972 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636966) roll requested 2024-11-19T17:50:36,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636972 2024-11-19T17:50:36,977 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,978 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,978 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,978 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636966 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636972 2024-11-19T17:50:36,983 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,983 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,983 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,983 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636972) roll requested 2024-11-19T17:50:36,983 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636983 2024-11-19T17:50:36,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636972 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636983 2024-11-19T17:50:36,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,992 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,992 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,992 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636983) roll requested 2024-11-19T17:50:36,992 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038636992 2024-11-19T17:50:36,998 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,998 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,999 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:36,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636983 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636992 2024-11-19T17:50:36,999 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:36,999 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,999 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:36,999 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038636992) roll requested 2024-11-19T17:50:37,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637000 2024-11-19T17:50:37,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,013 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,013 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,013 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,013 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,014 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038636992 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637000 2024-11-19T17:50:37,014 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,014 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637000) roll requested 2024-11-19T17:50:37,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637014 2024-11-19T17:50:37,026 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,026 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,026 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,026 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,026 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,027 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637000 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637014 2024-11-19T17:50:37,027 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,027 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,027 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,027 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637014) roll requested 2024-11-19T17:50:37,028 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637027 2024-11-19T17:50:37,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,040 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637014 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637027 2024-11-19T17:50:37,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,041 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637027) roll requested 2024-11-19T17:50:37,042 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637041 2024-11-19T17:50:37,048 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,048 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,048 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,048 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,048 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,048 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637027 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637041 2024-11-19T17:50:37,049 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,049 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,049 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637041) roll requested 2024-11-19T17:50:37,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637049 2024-11-19T17:50:37,055 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,055 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,056 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,056 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,056 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637041 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637049 2024-11-19T17:50:37,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,056 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637049) roll requested 2024-11-19T17:50:37,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637057 2024-11-19T17:50:37,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,063 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,064 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637049 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637057 2024-11-19T17:50:37,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,064 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637057) roll requested 2024-11-19T17:50:37,065 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637064 2024-11-19T17:50:37,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,070 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,070 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,070 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,070 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637057 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637064 2024-11-19T17:50:37,071 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,071 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,071 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,071 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637064) roll requested 2024-11-19T17:50:37,071 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637071 2024-11-19T17:50:37,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637064 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637071 2024-11-19T17:50:37,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,077 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637071) roll requested 2024-11-19T17:50:37,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637077 2024-11-19T17:50:37,086 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,086 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,086 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,087 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,087 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,087 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637071 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637077 2024-11-19T17:50:37,087 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,087 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,088 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,088 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637077) roll requested 2024-11-19T17:50:37,088 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637088 2024-11-19T17:50:37,094 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,094 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,094 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,094 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,094 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,094 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637077 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637088 2024-11-19T17:50:37,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,096 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637088) roll requested 2024-11-19T17:50:37,096 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637096 2024-11-19T17:50:37,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,104 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637088 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637096 2024-11-19T17:50:37,104 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,104 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637096) roll requested 2024-11-19T17:50:37,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637105 2024-11-19T17:50:37,110 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,111 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,111 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,111 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,111 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637096 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637105 2024-11-19T17:50:37,111 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,111 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,112 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,112 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637105) roll requested 2024-11-19T17:50:37,112 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637112 2024-11-19T17:50:37,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,122 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,122 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,122 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,122 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,122 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637105 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637112 2024-11-19T17:50:37,123 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,123 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,123 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,123 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637112) roll requested 2024-11-19T17:50:37,123 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637123 2024-11-19T17:50:37,129 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,129 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,130 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,130 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,130 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,130 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637112 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637123 2024-11-19T17:50:37,132 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,132 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,132 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,132 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637123) roll requested 2024-11-19T17:50:37,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637132 2024-11-19T17:50:37,138 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,138 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,138 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,138 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,138 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637123 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637132 2024-11-19T17:50:37,139 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,139 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,139 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,139 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637132) roll requested 2024-11-19T17:50:37,139 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637139 2024-11-19T17:50:37,144 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,144 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,144 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,144 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,145 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,145 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637132 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637139 2024-11-19T17:50:37,145 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,145 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637139) roll requested 2024-11-19T17:50:37,146 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637146 2024-11-19T17:50:37,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,151 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637139 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637146 2024-11-19T17:50:37,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,152 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,152 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637146) roll requested 2024-11-19T17:50:37,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637152 2024-11-19T17:50:37,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637146 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637152 2024-11-19T17:50:37,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,159 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,159 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637152) roll requested 2024-11-19T17:50:37,159 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637159 2024-11-19T17:50:37,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,170 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,170 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,170 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,170 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637152 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637159 2024-11-19T17:50:37,171 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,171 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,171 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,171 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637159) roll requested 2024-11-19T17:50:37,171 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637171 2024-11-19T17:50:37,176 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,176 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,176 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,176 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,177 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,177 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637159 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637171 2024-11-19T17:50:37,177 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,177 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637171) roll requested 2024-11-19T17:50:37,178 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637177 2024-11-19T17:50:37,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,182 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637171 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637177 2024-11-19T17:50:37,183 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,183 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,183 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,183 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637177) roll requested 2024-11-19T17:50:37,183 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637183 2024-11-19T17:50:37,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,188 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637177 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637183 2024-11-19T17:50:37,189 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,189 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,190 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637183) roll requested 2024-11-19T17:50:37,190 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637190 2024-11-19T17:50:37,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637183 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637190 2024-11-19T17:50:37,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,196 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637190) roll requested 2024-11-19T17:50:37,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637197 2024-11-19T17:50:37,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,202 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,202 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,202 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637190 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637197 2024-11-19T17:50:37,203 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,203 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,203 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,203 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637197) roll requested 2024-11-19T17:50:37,203 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637203 2024-11-19T17:50:37,209 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637197 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637203 2024-11-19T17:50:37,211 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,211 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,211 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637203) roll requested 2024-11-19T17:50:37,211 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637211 2024-11-19T17:50:37,216 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,216 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,216 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,216 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,216 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,216 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637203 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637211 2024-11-19T17:50:37,217 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,217 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,217 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637211) roll requested 2024-11-19T17:50:37,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637217 2024-11-19T17:50:37,222 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,222 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,222 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,223 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637211 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637217 2024-11-19T17:50:37,226 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,226 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,226 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,227 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637217) roll requested 2024-11-19T17:50:37,227 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637227 2024-11-19T17:50:37,233 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,233 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,233 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637217 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637227 2024-11-19T17:50:37,234 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,234 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,234 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,234 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637227) roll requested 2024-11-19T17:50:37,234 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637234 2024-11-19T17:50:37,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,239 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,239 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,239 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,239 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637227 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637234 2024-11-19T17:50:37,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,240 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,240 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637234) roll requested 2024-11-19T17:50:37,240 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637240 2024-11-19T17:50:37,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,246 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637234 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637240 2024-11-19T17:50:37,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,247 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,247 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,247 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637240) roll requested 2024-11-19T17:50:37,247 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637247 2024-11-19T17:50:37,257 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,257 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,257 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637240 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637247 2024-11-19T17:50:37,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,259 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,260 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637247) roll requested 2024-11-19T17:50:37,260 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637260 2024-11-19T17:50:37,265 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,265 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,265 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,265 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637247 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637260 2024-11-19T17:50:37,266 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,266 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,266 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,266 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637260) roll requested 2024-11-19T17:50:37,266 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637266 2024-11-19T17:50:37,271 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,271 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,271 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,271 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637260 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637266 2024-11-19T17:50:37,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,272 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,272 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637266) roll requested 2024-11-19T17:50:37,272 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637272 2024-11-19T17:50:37,277 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,277 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,277 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637266 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637272 2024-11-19T17:50:37,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,279 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,279 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,279 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637272) roll requested 2024-11-19T17:50:37,280 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637279 2024-11-19T17:50:37,285 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,285 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,285 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,285 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,285 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,285 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637272 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637279 2024-11-19T17:50:37,286 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,286 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,286 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,286 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637279) roll requested 2024-11-19T17:50:37,286 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637286 2024-11-19T17:50:37,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637279 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637286 2024-11-19T17:50:37,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,293 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637286) roll requested 2024-11-19T17:50:37,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637293 2024-11-19T17:50:37,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637286 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637293 2024-11-19T17:50:37,299 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,299 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,299 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,299 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637293) roll requested 2024-11-19T17:50:37,300 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637299 2024-11-19T17:50:37,304 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,304 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,304 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637293 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637299 2024-11-19T17:50:37,305 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,305 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637299) roll requested 2024-11-19T17:50:37,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637305 2024-11-19T17:50:37,310 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,310 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637299 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637305 2024-11-19T17:50:37,311 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,311 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,311 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637305) roll requested 2024-11-19T17:50:37,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637312 2024-11-19T17:50:37,316 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,316 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637305 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637312 2024-11-19T17:50:37,317 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,317 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637312) roll requested 2024-11-19T17:50:37,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637317 2024-11-19T17:50:37,322 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,322 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,322 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,323 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,323 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,323 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637312 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637317 2024-11-19T17:50:37,323 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,323 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,323 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637317) roll requested 2024-11-19T17:50:37,324 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637324 2024-11-19T17:50:37,328 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,329 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637317 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637324 2024-11-19T17:50:37,329 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,329 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,329 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637324) roll requested 2024-11-19T17:50:37,330 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637330 2024-11-19T17:50:37,334 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,334 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,335 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,335 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,335 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,335 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637324 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637330 2024-11-19T17:50:37,335 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,335 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,335 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,335 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637330) roll requested 2024-11-19T17:50:37,336 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637336 2024-11-19T17:50:37,340 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,340 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,340 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,341 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,341 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637330 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637336 2024-11-19T17:50:37,341 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,341 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,341 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,341 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637336) roll requested 2024-11-19T17:50:37,342 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637341 2024-11-19T17:50:37,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,346 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,346 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637336 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637341 2024-11-19T17:50:37,347 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,347 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,347 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,347 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637341) roll requested 2024-11-19T17:50:37,347 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637347 2024-11-19T17:50:37,352 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,352 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,352 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,352 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,352 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637341 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637347 2024-11-19T17:50:37,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,354 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,354 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,354 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637347) roll requested 2024-11-19T17:50:37,354 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637354 2024-11-19T17:50:37,359 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,359 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,359 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,359 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,360 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,360 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637347 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637354 2024-11-19T17:50:37,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,360 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,361 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637354) roll requested 2024-11-19T17:50:37,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637361 2024-11-19T17:50:37,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637354 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637361 2024-11-19T17:50:37,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,372 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,372 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637361) roll requested 2024-11-19T17:50:37,372 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637372 2024-11-19T17:50:37,377 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,378 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637361 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637372 2024-11-19T17:50:37,383 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,383 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,383 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,383 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637372) roll requested 2024-11-19T17:50:37,383 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637383 2024-11-19T17:50:37,391 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,391 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,391 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,391 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,391 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,391 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637372 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637383 2024-11-19T17:50:37,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,392 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,392 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,392 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637383) roll requested 2024-11-19T17:50:37,392 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637392 2024-11-19T17:50:37,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,401 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,401 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,401 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,401 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637383 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637392 2024-11-19T17:50:37,402 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,402 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,402 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,402 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637392) roll requested 2024-11-19T17:50:37,402 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637402 2024-11-19T17:50:37,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,408 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,408 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,408 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637392 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637402 2024-11-19T17:50:37,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,408 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,408 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637402) roll requested 2024-11-19T17:50:37,409 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637409 2024-11-19T17:50:37,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,414 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637402 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637409 2024-11-19T17:50:37,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,415 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637409) roll requested 2024-11-19T17:50:37,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637415 2024-11-19T17:50:37,420 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,421 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,421 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,421 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637409 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637415 2024-11-19T17:50:37,421 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,421 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,422 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,422 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637415) roll requested 2024-11-19T17:50:37,422 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637422 2024-11-19T17:50:37,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637415 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637422 2024-11-19T17:50:37,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,427 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637422) roll requested 2024-11-19T17:50:37,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637428 2024-11-19T17:50:37,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,434 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,434 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637422 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637428 2024-11-19T17:50:37,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,434 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,434 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637428) roll requested 2024-11-19T17:50:37,435 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637434 2024-11-19T17:50:37,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,440 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637428 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637434 2024-11-19T17:50:37,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,441 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637434) roll requested 2024-11-19T17:50:37,441 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637441 2024-11-19T17:50:37,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,447 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,447 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,447 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637434 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637441 2024-11-19T17:50:37,447 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,447 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,447 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,447 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637441) roll requested 2024-11-19T17:50:37,448 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637448 2024-11-19T17:50:37,452 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,452 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,452 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,453 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,453 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,453 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637441 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637448 2024-11-19T17:50:37,453 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,453 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,453 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,453 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637448) roll requested 2024-11-19T17:50:37,454 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637454 2024-11-19T17:50:37,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,459 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,459 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,459 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637448 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637454 2024-11-19T17:50:37,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,460 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,460 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637454) roll requested 2024-11-19T17:50:37,460 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637460 2024-11-19T17:50:37,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637454 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637460 2024-11-19T17:50:37,466 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,466 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637460) roll requested 2024-11-19T17:50:37,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637466 2024-11-19T17:50:37,472 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,472 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,472 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,472 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637460 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637466 2024-11-19T17:50:37,478 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,478 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637466) roll requested 2024-11-19T17:50:37,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637478 2024-11-19T17:50:37,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,486 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637466 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637478 2024-11-19T17:50:37,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,492 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,492 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,492 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637478) roll requested 2024-11-19T17:50:37,492 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637492 2024-11-19T17:50:37,504 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,504 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,504 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,504 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,504 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,504 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637478 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637492 2024-11-19T17:50:37,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,508 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,509 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637492) roll requested 2024-11-19T17:50:37,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637509 2024-11-19T17:50:37,515 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,515 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,515 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,515 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,515 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,516 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637492 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637509 2024-11-19T17:50:37,516 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,516 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,516 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,516 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637509) roll requested 2024-11-19T17:50:37,517 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637516 2024-11-19T17:50:37,523 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,523 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,523 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637509 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637516 2024-11-19T17:50:37,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,524 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,524 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,524 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637516) roll requested 2024-11-19T17:50:37,524 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637524 2024-11-19T17:50:37,535 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,535 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,536 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,536 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,536 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,536 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637516 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637524 2024-11-19T17:50:37,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,536 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637524) roll requested 2024-11-19T17:50:37,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637537 2024-11-19T17:50:37,544 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,544 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,544 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,544 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,544 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,544 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637524 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637537 2024-11-19T17:50:37,547 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,547 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637537) roll requested 2024-11-19T17:50:37,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637548 2024-11-19T17:50:37,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637537 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637548 2024-11-19T17:50:37,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,564 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,564 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637548) roll requested 2024-11-19T17:50:37,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637565 2024-11-19T17:50:37,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,578 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,578 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637548 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637565 2024-11-19T17:50:37,581 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,582 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,582 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,582 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637565) roll requested 2024-11-19T17:50:37,583 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637583 2024-11-19T17:50:37,608 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,608 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,611 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,611 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,611 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637565 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637583 2024-11-19T17:50:37,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,612 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,613 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637583) roll requested 2024-11-19T17:50:37,613 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637613 2024-11-19T17:50:37,632 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,633 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,633 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637583 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637613 2024-11-19T17:50:37,648 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,648 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,648 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637613) roll requested 2024-11-19T17:50:37,649 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637649 2024-11-19T17:50:37,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,662 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,662 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,662 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,662 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,662 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637613 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637649 2024-11-19T17:50:37,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,680 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,680 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637649) roll requested 2024-11-19T17:50:37,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637681 2024-11-19T17:50:37,692 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,692 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,692 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,692 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,692 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,692 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637649 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637681 2024-11-19T17:50:37,693 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,693 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,694 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,694 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637681) roll requested 2024-11-19T17:50:37,694 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637694 2024-11-19T17:50:37,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,709 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637681 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637694 2024-11-19T17:50:37,709 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,709 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637694) roll requested 2024-11-19T17:50:37,710 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637710 2024-11-19T17:50:37,720 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,720 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,721 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,721 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,721 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,721 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637694 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637710 2024-11-19T17:50:37,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,724 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,724 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637710) roll requested 2024-11-19T17:50:37,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637725 2024-11-19T17:50:37,735 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,735 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,735 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,735 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,736 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,736 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637710 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637725 2024-11-19T17:50:37,737 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,737 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,737 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637725) roll requested 2024-11-19T17:50:37,738 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637737 2024-11-19T17:50:37,760 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,760 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,760 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,760 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,761 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,761 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637725 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637737 2024-11-19T17:50:37,761 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,761 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,761 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,762 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637737) roll requested 2024-11-19T17:50:37,762 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637762 2024-11-19T17:50:37,768 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,768 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,768 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,769 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,769 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,769 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637737 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637762 2024-11-19T17:50:37,770 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,770 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,770 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,770 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637762) roll requested 2024-11-19T17:50:37,770 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637770 2024-11-19T17:50:37,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,777 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637762 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637770 2024-11-19T17:50:37,780 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,781 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,781 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,781 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637770) roll requested 2024-11-19T17:50:37,781 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637781 2024-11-19T17:50:37,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,795 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,795 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637770 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637781 2024-11-19T17:50:37,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,796 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637781) roll requested 2024-11-19T17:50:37,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637797 2024-11-19T17:50:37,810 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,810 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,810 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,820 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,820 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637781 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637797 2024-11-19T17:50:37,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,828 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,829 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637797) roll requested 2024-11-19T17:50:37,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637829 2024-11-19T17:50:37,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,837 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,837 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,837 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,837 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637797 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637829 2024-11-19T17:50:37,837 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,837 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,837 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637829) roll requested 2024-11-19T17:50:37,838 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637838 2024-11-19T17:50:37,853 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637829 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637838 2024-11-19T17:50:37,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,865 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,865 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637838) roll requested 2024-11-19T17:50:37,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637865 2024-11-19T17:50:37,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,875 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,875 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,875 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,875 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637838 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637865 2024-11-19T17:50:37,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,876 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637865) roll requested 2024-11-19T17:50:37,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637876 2024-11-19T17:50:37,887 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,888 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,888 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,888 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,888 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,888 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637865 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637876 2024-11-19T17:50:37,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,893 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637876) roll requested 2024-11-19T17:50:37,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637893 2024-11-19T17:50:37,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637876 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637893 2024-11-19T17:50:37,904 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,904 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,904 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,904 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637893) roll requested 2024-11-19T17:50:37,905 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637905 2024-11-19T17:50:37,911 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,911 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,911 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,911 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,911 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,911 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637893 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637905 2024-11-19T17:50:37,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,912 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,912 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,912 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637905) roll requested 2024-11-19T17:50:37,913 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637912 2024-11-19T17:50:37,919 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,919 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,919 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637905 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637912 2024-11-19T17:50:37,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,920 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637912) roll requested 2024-11-19T17:50:37,920 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637920 2024-11-19T17:50:37,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,927 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637912 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637920 2024-11-19T17:50:37,928 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,928 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,928 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637920) roll requested 2024-11-19T17:50:37,929 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637929 2024-11-19T17:50:37,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,940 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,940 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,940 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,940 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637920 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637929 2024-11-19T17:50:37,942 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,942 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,942 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,942 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637929) roll requested 2024-11-19T17:50:37,942 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637942 2024-11-19T17:50:37,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,958 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,959 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637929 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637942 2024-11-19T17:50:37,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,960 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637942) roll requested 2024-11-19T17:50:37,961 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637960 2024-11-19T17:50:37,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,971 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,971 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,971 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637942 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637960 2024-11-19T17:50:37,979 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,979 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,979 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637960) roll requested 2024-11-19T17:50:37,979 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637979 2024-11-19T17:50:37,992 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,993 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,993 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,993 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,993 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:37,993 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637960 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637979 2024-11-19T17:50:37,994 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:37,994 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,994 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:37,994 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637979) roll requested 2024-11-19T17:50:37,995 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038637994 2024-11-19T17:50:38,008 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,008 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,009 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637979 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637994 2024-11-19T17:50:38,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,020 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,020 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,021 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038637994) roll requested 2024-11-19T17:50:38,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638021 2024-11-19T17:50:38,028 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,028 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,029 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,029 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,029 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038637994 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638021 2024-11-19T17:50:38,031 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,031 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638021) roll requested 2024-11-19T17:50:38,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638032 2024-11-19T17:50:38,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,043 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,060 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,060 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638021 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638032 2024-11-19T17:50:38,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,072 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638032) roll requested 2024-11-19T17:50:38,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638073 2024-11-19T17:50:38,090 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,091 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,091 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638032 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638073 2024-11-19T17:50:38,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,098 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,098 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,098 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638073) roll requested 2024-11-19T17:50:38,099 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638099 2024-11-19T17:50:38,121 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,121 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,121 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,121 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,121 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,121 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638073 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638099 2024-11-19T17:50:38,122 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,122 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,122 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,122 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638099) roll requested 2024-11-19T17:50:38,123 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638122 2024-11-19T17:50:38,132 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,132 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,133 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,133 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,136 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,136 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638099 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638122 2024-11-19T17:50:38,137 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,137 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,137 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,137 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638122) roll requested 2024-11-19T17:50:38,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638137 2024-11-19T17:50:38,152 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,152 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,152 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,153 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638122 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638137 2024-11-19T17:50:38,157 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,157 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,157 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638137) roll requested 2024-11-19T17:50:38,158 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638158 2024-11-19T17:50:38,168 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,169 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,169 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,169 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,169 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,169 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638137 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638158 2024-11-19T17:50:38,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,172 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638158) roll requested 2024-11-19T17:50:38,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638173 2024-11-19T17:50:38,205 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,205 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,205 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,205 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,205 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,205 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638158 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638173 2024-11-19T17:50:38,212 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,212 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,212 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,212 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638173) roll requested 2024-11-19T17:50:38,213 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638213 2024-11-19T17:50:38,220 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,221 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,221 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,221 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638173 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638213 2024-11-19T17:50:38,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,224 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,224 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,224 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638213) roll requested 2024-11-19T17:50:38,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638225 2024-11-19T17:50:38,235 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,235 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,235 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,235 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,235 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,235 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638213 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638225 2024-11-19T17:50:38,236 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,236 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,236 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,236 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638225) roll requested 2024-11-19T17:50:38,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638236 2024-11-19T17:50:38,242 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,242 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,242 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,242 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,242 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,242 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638225 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638236 2024-11-19T17:50:38,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,244 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,244 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638236) roll requested 2024-11-19T17:50:38,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638244 2024-11-19T17:50:38,254 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,254 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,254 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,254 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,254 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,255 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638236 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638244 2024-11-19T17:50:38,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,256 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,256 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,256 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638244) roll requested 2024-11-19T17:50:38,256 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638256 2024-11-19T17:50:38,270 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,271 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,271 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,271 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638244 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638256 2024-11-19T17:50:38,273 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,273 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,273 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638256) roll requested 2024-11-19T17:50:38,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638273 2024-11-19T17:50:38,279 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,279 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,279 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,279 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,279 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,279 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638256 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638273 2024-11-19T17:50:38,280 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,280 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,280 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,280 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638273) roll requested 2024-11-19T17:50:38,280 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638280 2024-11-19T17:50:38,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,286 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,286 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638273 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638280 2024-11-19T17:50:38,287 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,287 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638280) roll requested 2024-11-19T17:50:38,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638288 2024-11-19T17:50:38,294 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,294 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,294 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,294 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,295 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638280 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638288 2024-11-19T17:50:38,295 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,295 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,296 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,296 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638288) roll requested 2024-11-19T17:50:38,296 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638296 2024-11-19T17:50:38,302 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,302 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,302 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,302 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,302 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,302 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638288 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638296 2024-11-19T17:50:38,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,304 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638296) roll requested 2024-11-19T17:50:38,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638305 2024-11-19T17:50:38,310 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,310 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,310 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,310 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638296 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638305 2024-11-19T17:50:38,311 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,311 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638305) roll requested 2024-11-19T17:50:38,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638312 2024-11-19T17:50:38,318 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,318 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,318 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,319 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638305 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638312 2024-11-19T17:50:38,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,320 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638312) roll requested 2024-11-19T17:50:38,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638320 2024-11-19T17:50:38,333 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638312 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638320 2024-11-19T17:50:38,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,334 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638320) roll requested 2024-11-19T17:50:38,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638334 2024-11-19T17:50:38,344 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,344 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,344 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,344 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,344 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,344 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638320 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638334 2024-11-19T17:50:38,345 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,345 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,345 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,345 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638334) roll requested 2024-11-19T17:50:38,345 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638345 2024-11-19T17:50:38,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,360 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,360 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638334 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638345 2024-11-19T17:50:38,365 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,365 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,365 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,365 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638345) roll requested 2024-11-19T17:50:38,365 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638365 2024-11-19T17:50:38,370 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,370 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,370 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,370 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638345 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638365 2024-11-19T17:50:38,371 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,371 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,372 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638365) roll requested 2024-11-19T17:50:38,372 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638372 2024-11-19T17:50:38,377 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,378 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638365 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638372 2024-11-19T17:50:38,378 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,378 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,379 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,379 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638372) roll requested 2024-11-19T17:50:38,379 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638379 2024-11-19T17:50:38,385 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,385 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,385 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,385 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,385 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,385 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638372 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638379 2024-11-19T17:50:38,393 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,393 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,393 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,393 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638379) roll requested 2024-11-19T17:50:38,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638393 2024-11-19T17:50:38,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,403 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,404 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638379 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638393 2024-11-19T17:50:38,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,408 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,408 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638393) roll requested 2024-11-19T17:50:38,409 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638408 2024-11-19T17:50:38,418 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,418 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638393 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638408 2024-11-19T17:50:38,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,420 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638408) roll requested 2024-11-19T17:50:38,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638421 2024-11-19T17:50:38,428 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,428 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,428 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,428 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,428 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,428 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638408 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638421 2024-11-19T17:50:38,429 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,429 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,430 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,430 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638421) roll requested 2024-11-19T17:50:38,430 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638430 2024-11-19T17:50:38,435 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,436 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,436 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,436 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,436 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,436 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638421 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638430 2024-11-19T17:50:38,436 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,436 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,437 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638430) roll requested 2024-11-19T17:50:38,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638437 2024-11-19T17:50:38,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638430 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638437 2024-11-19T17:50:38,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,449 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,449 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638437) roll requested 2024-11-19T17:50:38,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638449 2024-11-19T17:50:38,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,460 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,460 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638437 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638449 2024-11-19T17:50:38,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,464 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638449) roll requested 2024-11-19T17:50:38,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638465 2024-11-19T17:50:38,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,472 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638449 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638465 2024-11-19T17:50:38,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,473 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638465) roll requested 2024-11-19T17:50:38,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638473 2024-11-19T17:50:38,487 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,487 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,487 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,487 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638465 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638473 2024-11-19T17:50:38,489 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,489 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,490 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638473) roll requested 2024-11-19T17:50:38,490 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638490 2024-11-19T17:50:38,499 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638473 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638490 2024-11-19T17:50:38,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,501 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638490) roll requested 2024-11-19T17:50:38,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638501 2024-11-19T17:50:38,510 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,510 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,510 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,510 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,510 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638490 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638501 2024-11-19T17:50:38,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,512 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,512 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638501) roll requested 2024-11-19T17:50:38,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638513 2024-11-19T17:50:38,519 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,520 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,525 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,525 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,525 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638501 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638513 2024-11-19T17:50:38,526 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,526 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,526 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,526 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638513) roll requested 2024-11-19T17:50:38,526 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638526 2024-11-19T17:50:38,541 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,541 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,541 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,542 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638513 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638526 2024-11-19T17:50:38,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,544 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,545 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,545 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638526) roll requested 2024-11-19T17:50:38,545 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638545 2024-11-19T17:50:38,552 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,552 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,552 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,552 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,552 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,552 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638526 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638545 2024-11-19T17:50:38,553 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,553 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,553 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638545) roll requested 2024-11-19T17:50:38,554 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638553 2024-11-19T17:50:38,564 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,565 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,565 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,565 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,565 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,565 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638545 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638553 2024-11-19T17:50:38,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,569 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,569 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,569 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638553) roll requested 2024-11-19T17:50:38,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638569 2024-11-19T17:50:38,579 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,579 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,579 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638553 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638569 2024-11-19T17:50:38,581 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,583 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,583 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,583 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638569) roll requested 2024-11-19T17:50:38,584 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638583 2024-11-19T17:50:38,593 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,593 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,593 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,593 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,593 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,593 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638569 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638583 2024-11-19T17:50:38,594 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,594 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,594 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,594 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638583) roll requested 2024-11-19T17:50:38,594 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638594 2024-11-19T17:50:38,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,601 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638583 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638594 2024-11-19T17:50:38,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,604 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,605 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638594) roll requested 2024-11-19T17:50:38,605 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638605 2024-11-19T17:50:38,611 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,611 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,611 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,612 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,612 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,612 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638594 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638605 2024-11-19T17:50:38,615 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,615 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,615 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,615 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638605) roll requested 2024-11-19T17:50:38,616 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638615 2024-11-19T17:50:38,627 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,629 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,629 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,629 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638605 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638615 2024-11-19T17:50:38,630 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,630 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,630 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,630 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638615) roll requested 2024-11-19T17:50:38,630 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638630 2024-11-19T17:50:38,636 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,636 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,636 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,636 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,636 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,636 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638615 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638630 2024-11-19T17:50:38,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,637 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,638 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638630) roll requested 2024-11-19T17:50:38,638 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638638 2024-11-19T17:50:38,648 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,648 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,648 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,649 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,649 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,649 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638630 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638638 2024-11-19T17:50:38,653 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,653 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,653 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,653 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638638) roll requested 2024-11-19T17:50:38,653 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638653 2024-11-19T17:50:38,659 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,660 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,660 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,660 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638638 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638653 2024-11-19T17:50:38,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,664 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,665 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,665 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638653) roll requested 2024-11-19T17:50:38,665 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638665 2024-11-19T17:50:38,679 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,679 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,679 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,679 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,679 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,679 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638653 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638665 2024-11-19T17:50:38,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,680 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638665) roll requested 2024-11-19T17:50:38,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638681 2024-11-19T17:50:38,702 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638665 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638681 2024-11-19T17:50:38,712 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,712 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,712 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,712 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638681) roll requested 2024-11-19T17:50:38,713 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638712 2024-11-19T17:50:38,728 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,728 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,728 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,729 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,729 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,729 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638681 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638712 2024-11-19T17:50:38,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,736 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638712) roll requested 2024-11-19T17:50:38,737 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638737 2024-11-19T17:50:38,746 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,746 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,747 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,747 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,747 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,747 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638712 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638737 2024-11-19T17:50:38,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,752 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638737) roll requested 2024-11-19T17:50:38,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638753 2024-11-19T17:50:38,767 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638737 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638753 2024-11-19T17:50:38,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,768 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638753) roll requested 2024-11-19T17:50:38,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638769 2024-11-19T17:50:38,776 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,776 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,777 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,777 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638753 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638769 2024-11-19T17:50:38,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,778 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,778 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638769) roll requested 2024-11-19T17:50:38,778 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638778 2024-11-19T17:50:38,794 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,794 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,795 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,795 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638769 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638778 2024-11-19T17:50:38,795 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,795 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,795 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638778) roll requested 2024-11-19T17:50:38,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638796 2024-11-19T17:50:38,807 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,807 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,807 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,807 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,807 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,807 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638778 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638796 2024-11-19T17:50:38,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,808 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638796) roll requested 2024-11-19T17:50:38,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638808 2024-11-19T17:50:38,817 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,817 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,817 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,817 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,817 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,817 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638796 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638808 2024-11-19T17:50:38,818 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,818 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,818 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,818 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638808) roll requested 2024-11-19T17:50:38,818 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638818 2024-11-19T17:50:38,837 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,837 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,837 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,837 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,838 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,838 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638808 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638818 2024-11-19T17:50:38,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,840 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638818) roll requested 2024-11-19T17:50:38,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638841 2024-11-19T17:50:38,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,849 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,849 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,849 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638818 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638841 2024-11-19T17:50:38,851 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,851 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,851 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,851 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638841) roll requested 2024-11-19T17:50:38,851 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638851 2024-11-19T17:50:38,860 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,860 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,861 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,861 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,861 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,861 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638841 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638851 2024-11-19T17:50:38,862 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,862 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,862 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,862 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638851) roll requested 2024-11-19T17:50:38,862 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638862 2024-11-19T17:50:38,868 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,868 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,868 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,869 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,869 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,869 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638851 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638862 2024-11-19T17:50:38,869 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,869 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,869 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,869 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638862) roll requested 2024-11-19T17:50:38,870 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638870 2024-11-19T17:50:38,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,901 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,901 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,901 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,901 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,901 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638862 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638870 2024-11-19T17:50:38,902 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,902 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,903 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,903 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638870) roll requested 2024-11-19T17:50:38,903 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638903 2024-11-19T17:50:38,910 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,910 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,910 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,910 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,910 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638870 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638903 2024-11-19T17:50:38,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,912 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,912 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,912 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638903) roll requested 2024-11-19T17:50:38,913 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638912 2024-11-19T17:50:38,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,925 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638903 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638912 2024-11-19T17:50:38,926 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,926 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,926 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,926 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638912) roll requested 2024-11-19T17:50:38,926 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638926 2024-11-19T17:50:38,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,939 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,939 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,939 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638912 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638926 2024-11-19T17:50:38,941 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,941 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,941 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,941 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638926) roll requested 2024-11-19T17:50:38,941 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638941 2024-11-19T17:50:38,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638926 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638941 2024-11-19T17:50:38,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,960 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638941) roll requested 2024-11-19T17:50:38,961 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638960 2024-11-19T17:50:38,967 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,968 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,968 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,968 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,968 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638941 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638960 2024-11-19T17:50:38,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,972 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638960) roll requested 2024-11-19T17:50:38,973 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638972 2024-11-19T17:50:38,987 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,987 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,988 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,988 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:38,988 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638960 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638972 2024-11-19T17:50:38,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:38,989 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,989 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:38,989 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638972) roll requested 2024-11-19T17:50:38,989 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038638989 2024-11-19T17:50:39,000 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,000 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,000 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,000 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,004 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,004 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638972 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638989 2024-11-19T17:50:39,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:39,012 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038638989) roll requested 2024-11-19T17:50:39,013 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038639013 2024-11-19T17:50:39,020 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,020 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,020 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,020 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,020 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,020 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038638989 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639013 2024-11-19T17:50:39,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:39,021 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,022 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,022 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038639013) roll requested 2024-11-19T17:50:39,022 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038639022 2024-11-19T17:50:39,032 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,033 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,033 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,033 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,033 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,033 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639013 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639022 2024-11-19T17:50:39,033 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:39,033 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,034 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,034 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038639022) roll requested 2024-11-19T17:50:39,034 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038639034 2024-11-19T17:50:39,044 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,044 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,044 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,045 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,045 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,045 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639022 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639034 2024-11-19T17:50:39,046 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:39,046 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,046 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,046 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038639034) roll requested 2024-11-19T17:50:39,047 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038639047 2024-11-19T17:50:39,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,060 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639034 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639047 2024-11-19T17:50:39,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:39,064 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038639047) roll requested 2024-11-19T17:50:39,065 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038639065 2024-11-19T17:50:39,072 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,072 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,072 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,072 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,072 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,072 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639047 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639065 2024-11-19T17:50:39,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:39,083 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,083 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038639065) roll requested 2024-11-19T17:50:39,083 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038639083 2024-11-19T17:50:39,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,092 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,092 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,092 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,092 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,092 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639065 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639083 2024-11-19T17:50:39,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:39,092 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038639083) roll requested 2024-11-19T17:50:39,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038639093 2024-11-19T17:50:39,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639083 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639093 2024-11-19T17:50:39,108 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:39,108 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,108 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,108 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038639093) roll requested 2024-11-19T17:50:39,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038639108 2024-11-19T17:50:39,117 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,117 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,117 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,117 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,118 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639093 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639108 2024-11-19T17:50:39,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:39,120 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038639108) roll requested 2024-11-19T17:50:39,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038639121 2024-11-19T17:50:39,140 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,140 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,140 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,140 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,140 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,140 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639108 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639121 2024-11-19T17:50:39,142 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:39,142 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,142 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,142 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038639121) roll requested 2024-11-19T17:50:39,143 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038639142 2024-11-19T17:50:39,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639121 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639142 2024-11-19T17:50:39,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:39,165 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,166 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038639142) roll requested 2024-11-19T17:50:39,166 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038639166 2024-11-19T17:50:39,180 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,180 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,181 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,181 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,181 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,182 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639142 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639166 2024-11-19T17:50:39,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:39,188 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038639166) roll requested 2024-11-19T17:50:39,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038639189 2024-11-19T17:50:39,208 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,208 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,208 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,208 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,208 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,208 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639166 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639189 2024-11-19T17:50:39,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:39,224 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038639189) roll requested 2024-11-19T17:50:39,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038639225 2024-11-19T17:50:39,233 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,233 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,233 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,234 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,234 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,234 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639189 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639225 2024-11-19T17:50:39,236 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:39,236 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,236 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038639225) roll requested 2024-11-19T17:50:39,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038639237 2024-11-19T17:50:39,244 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,244 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639225 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639237 2024-11-19T17:50:39,248 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:39,248 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,248 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,248 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038639237) roll requested 2024-11-19T17:50:39,249 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038639248 2024-11-19T17:50:39,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,256 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,256 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,256 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639237 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639248 2024-11-19T17:50:39,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:39,257 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038639248) roll requested 2024-11-19T17:50:39,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038639257 2024-11-19T17:50:39,264 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,264 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,264 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,264 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,268 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,268 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639248 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639257 2024-11-19T17:50:39,269 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:39,269 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,269 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038639257) roll requested 2024-11-19T17:50:39,270 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038639269 2024-11-19T17:50:39,281 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,281 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,282 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,282 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,282 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,282 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639257 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639269 2024-11-19T17:50:39,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:39,292 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038639269) roll requested 2024-11-19T17:50:39,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038639293 2024-11-19T17:50:39,302 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,302 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,302 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,303 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,303 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639269 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639293 2024-11-19T17:50:39,303 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:39,303 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,303 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038639293) roll requested 2024-11-19T17:50:39,304 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038639304 2024-11-19T17:50:39,332 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639293 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639304 2024-11-19T17:50:39,335 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:39,335 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,335 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,335 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038639304) roll requested 2024-11-19T17:50:39,335 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038639335 2024-11-19T17:50:39,343 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,344 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639304 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639335 2024-11-19T17:50:39,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:39,348 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,348 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038639335) roll requested 2024-11-19T17:50:39,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038639349 2024-11-19T17:50:39,362 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,363 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,363 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,363 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,363 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639335 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639349 2024-11-19T17:50:39,371 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:39,371 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038639349) roll requested 2024-11-19T17:50:39,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038639371 2024-11-19T17:50:39,385 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,385 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,385 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,385 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,385 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,385 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639349 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639371 2024-11-19T17:50:39,393 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:39,393 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,393 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,393 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038639371) roll requested 2024-11-19T17:50:39,394 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038639394 2024-11-19T17:50:39,404 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,404 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,404 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,404 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639371 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639394 2024-11-19T17:50:39,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:39,405 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038639394) roll requested 2024-11-19T17:50:39,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038639406 2024-11-19T17:50:39,428 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,429 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,429 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,429 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639394 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639406 2024-11-19T17:50:39,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:39,434 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,435 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038639406) roll requested 2024-11-19T17:50:39,436 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038639435 2024-11-19T17:50:39,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,457 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,459 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,459 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,459 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639406 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639435 2024-11-19T17:50:39,461 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:39,461 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038639435) roll requested 2024-11-19T17:50:39,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038639461 2024-11-19T17:50:39,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639435 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639461 2024-11-19T17:50:39,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:39,492 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,492 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,492 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038639461) roll requested 2024-11-19T17:50:39,492 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038639492 2024-11-19T17:50:39,499 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639461 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639492 2024-11-19T17:50:39,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:39,500 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038639492) roll requested 2024-11-19T17:50:39,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038639501 2024-11-19T17:50:39,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639492 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639501 2024-11-19T17:50:39,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:39,524 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,524 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,524 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038639501) roll requested 2024-11-19T17:50:39,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038639524 2024-11-19T17:50:39,547 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,547 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,547 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,548 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,548 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639501 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639524 2024-11-19T17:50:39,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:39,548 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038639524) roll requested 2024-11-19T17:50:39,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038639549 2024-11-19T17:50:39,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,572 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,572 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,573 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,573 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,573 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639524 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639549 2024-11-19T17:50:39,577 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:39,577 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038639549) roll requested 2024-11-19T17:50:39,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038639577 2024-11-19T17:50:39,587 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639549 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639577 2024-11-19T17:50:39,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:39,601 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038639577) roll requested 2024-11-19T17:50:39,602 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038639602 2024-11-19T17:50:39,615 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,615 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,616 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,616 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,616 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639577 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639602 2024-11-19T17:50:39,616 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:39,616 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,617 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038639602) roll requested 2024-11-19T17:50:39,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038639617 2024-11-19T17:50:39,623 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,624 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,624 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,624 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,624 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639602 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639617 2024-11-19T17:50:39,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:39,624 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038639617) roll requested 2024-11-19T17:50:39,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038639625 2024-11-19T17:50:39,632 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,632 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,633 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,633 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639617 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639625 2024-11-19T17:50:39,636 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:39,636 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,636 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,636 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038639625) roll requested 2024-11-19T17:50:39,637 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038639636 2024-11-19T17:50:39,648 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,648 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,649 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,655 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639625 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639636 2024-11-19T17:50:39,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:39,656 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038639636) roll requested 2024-11-19T17:50:39,657 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038639657 2024-11-19T17:50:39,666 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,666 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,667 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,667 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,667 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639636 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639657 2024-11-19T17:50:39,668 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:39,669 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,669 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038639657) roll requested 2024-11-19T17:50:39,669 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038639669 2024-11-19T17:50:39,676 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,680 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,680 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,680 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639657 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639669 2024-11-19T17:50:39,682 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:39,682 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,682 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,682 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038639669) roll requested 2024-11-19T17:50:39,682 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038639682 2024-11-19T17:50:39,690 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,691 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,691 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,691 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,691 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,691 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639669 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639682 2024-11-19T17:50:39,693 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:39,693 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,693 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,693 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038639682) roll requested 2024-11-19T17:50:39,694 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038639693 2024-11-19T17:50:39,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639682 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639693 2024-11-19T17:50:39,704 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:39,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038639693) roll requested 2024-11-19T17:50:39,705 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038639705 2024-11-19T17:50:39,716 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,719 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,719 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,719 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,720 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639693 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639705 2024-11-19T17:50:39,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:39,724 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,724 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038639705) roll requested 2024-11-19T17:50:39,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038639725 2024-11-19T17:50:39,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,750 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,750 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,751 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639705 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639725 2024-11-19T17:50:39,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:39,751 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,751 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,751 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038639725) roll requested 2024-11-19T17:50:39,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038639751 2024-11-19T17:50:39,759 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,760 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,760 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,760 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639725 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639751 2024-11-19T17:50:39,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:39,760 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,761 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,761 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038639751) roll requested 2024-11-19T17:50:39,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038639761 2024-11-19T17:50:39,773 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,774 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,774 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,774 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,774 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,774 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639751 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639761 2024-11-19T17:50:39,777 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:39,777 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,777 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,778 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038639761) roll requested 2024-11-19T17:50:39,778 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038639778 2024-11-19T17:50:39,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,794 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639761 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639778 2024-11-19T17:50:39,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:39,806 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,807 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,807 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038639778) roll requested 2024-11-19T17:50:39,807 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038639807 2024-11-19T17:50:39,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,816 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,816 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639778 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639807 2024-11-19T17:50:39,817 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:39,817 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,817 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,817 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038639807) roll requested 2024-11-19T17:50:39,818 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038639818 2024-11-19T17:50:39,824 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,825 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,825 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,825 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,825 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639807 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639818 2024-11-19T17:50:39,830 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:39,830 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,831 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038639818) roll requested 2024-11-19T17:50:39,831 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038639831 2024-11-19T17:50:39,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,852 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,852 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,852 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,852 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,852 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639818 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639831 2024-11-19T17:50:39,853 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:39,853 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,853 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,853 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038639831) roll requested 2024-11-19T17:50:39,853 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038639853 2024-11-19T17:50:39,864 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,865 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,865 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,865 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,865 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,865 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639831 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639853 2024-11-19T17:50:39,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:39,868 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,869 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038639853) roll requested 2024-11-19T17:50:39,869 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038639869 2024-11-19T17:50:39,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,881 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,881 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,881 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,881 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639853 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639869 2024-11-19T17:50:39,882 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:39,883 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,883 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038639869) roll requested 2024-11-19T17:50:39,883 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038639883 2024-11-19T17:50:39,901 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,901 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,902 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,902 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,902 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,902 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639869 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639883 2024-11-19T17:50:39,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:39,917 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,918 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,918 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038639883) roll requested 2024-11-19T17:50:39,918 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038639918 2024-11-19T17:50:39,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639883 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639918 2024-11-19T17:50:39,937 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:39,938 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,938 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,938 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038639918) roll requested 2024-11-19T17:50:39,938 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038639938 2024-11-19T17:50:39,964 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,964 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,965 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,965 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:39,965 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639918 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639938 2024-11-19T17:50:39,966 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:39,966 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,966 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:39,966 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038639938) roll requested 2024-11-19T17:50:39,966 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038639966 2024-11-19T17:50:40,008 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,008 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,009 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639938 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639966 2024-11-19T17:50:40,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:40,020 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,020 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,021 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038639966) roll requested 2024-11-19T17:50:40,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038640021 2024-11-19T17:50:40,028 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,028 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,028 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,029 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,029 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038639966 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640021 2024-11-19T17:50:40,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:40,032 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038640021) roll requested 2024-11-19T17:50:40,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038640033 2024-11-19T17:50:40,045 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,048 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,048 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,048 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,048 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640021 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640033 2024-11-19T17:50:40,049 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:40,049 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,049 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038640033) roll requested 2024-11-19T17:50:40,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038640049 2024-11-19T17:50:40,068 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,068 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640033 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640049 2024-11-19T17:50:40,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:40,076 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,076 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,076 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038640049) roll requested 2024-11-19T17:50:40,076 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038640076 2024-11-19T17:50:40,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,091 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,091 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640049 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640076 2024-11-19T17:50:40,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:40,092 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,092 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038640076) roll requested 2024-11-19T17:50:40,092 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038640092 2024-11-19T17:50:40,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640076 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640092 2024-11-19T17:50:40,107 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:40,107 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,107 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,108 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038640092) roll requested 2024-11-19T17:50:40,108 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038640108 2024-11-19T17:50:40,125 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,125 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,125 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,125 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,125 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,125 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640092 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640108 2024-11-19T17:50:40,126 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:40,126 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,126 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,126 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038640108) roll requested 2024-11-19T17:50:40,127 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038640126 2024-11-19T17:50:40,132 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,132 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,132 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,133 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,133 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,133 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640108 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640126 2024-11-19T17:50:40,133 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:40,133 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,133 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038640126) roll requested 2024-11-19T17:50:40,134 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038640133 2024-11-19T17:50:40,143 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,143 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,143 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,143 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,143 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,143 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640126 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640133 2024-11-19T17:50:40,144 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:40,144 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,144 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038640133) roll requested 2024-11-19T17:50:40,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038640145 2024-11-19T17:50:40,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,152 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,152 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640133 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640145 2024-11-19T17:50:40,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:40,160 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038640145) roll requested 2024-11-19T17:50:40,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038640160 2024-11-19T17:50:40,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,168 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,168 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,168 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640145 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640160 2024-11-19T17:50:40,169 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:40,169 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,169 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,169 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038640160) roll requested 2024-11-19T17:50:40,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038640169 2024-11-19T17:50:40,186 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,186 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,186 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,186 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640160 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640169 2024-11-19T17:50:40,191 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:40,191 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,192 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,192 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038640169) roll requested 2024-11-19T17:50:40,193 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038640193 2024-11-19T17:50:40,205 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,205 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,205 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,206 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,211 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640169 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640193 2024-11-19T17:50:40,211 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,211 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:40,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,211 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038640193) roll requested 2024-11-19T17:50:40,212 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038640212 2024-11-19T17:50:40,233 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,233 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,233 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640193 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640212 2024-11-19T17:50:40,236 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:40,236 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,236 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038640212) roll requested 2024-11-19T17:50:40,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038640237 2024-11-19T17:50:40,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,256 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,256 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,256 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,256 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640212 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640237 2024-11-19T17:50:40,260 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:40,260 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038640237) roll requested 2024-11-19T17:50:40,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038640261 2024-11-19T17:50:40,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,269 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,269 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,269 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,269 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640237 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640261 2024-11-19T17:50:40,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:40,272 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038640261) roll requested 2024-11-19T17:50:40,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038640273 2024-11-19T17:50:40,280 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,280 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,280 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,280 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640261 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640273 2024-11-19T17:50:40,281 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:40,281 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,281 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,281 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038640273) roll requested 2024-11-19T17:50:40,282 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038640281 2024-11-19T17:50:40,287 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,288 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,288 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,288 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,288 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,288 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640273 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640281 2024-11-19T17:50:40,289 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:40,289 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,290 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,290 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038640281) roll requested 2024-11-19T17:50:40,290 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038640290 2024-11-19T17:50:40,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,299 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640281 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640290 2024-11-19T17:50:40,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:40,300 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,300 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038640290) roll requested 2024-11-19T17:50:40,300 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038640300 2024-11-19T17:50:40,307 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,307 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,307 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,307 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,307 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,308 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640290 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640300 2024-11-19T17:50:40,309 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:40,309 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,309 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,309 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038640300) roll requested 2024-11-19T17:50:40,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038640309 2024-11-19T17:50:40,320 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,321 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,321 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,321 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,321 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640300 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640309 2024-11-19T17:50:40,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:40,328 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,328 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,328 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038640309) roll requested 2024-11-19T17:50:40,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038640328 2024-11-19T17:50:40,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,340 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,340 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,341 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,341 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,341 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640309 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640328 2024-11-19T17:50:40,343 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:40,343 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,343 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,343 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038640328) roll requested 2024-11-19T17:50:40,343 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038640343 2024-11-19T17:50:40,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,379 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640328 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640343 2024-11-19T17:50:40,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:40,392 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,393 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,393 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038640343) roll requested 2024-11-19T17:50:40,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038640393 2024-11-19T17:50:40,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,400 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,400 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,400 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,400 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,400 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640343 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640393 2024-11-19T17:50:40,401 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:40,401 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,401 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,401 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038640393) roll requested 2024-11-19T17:50:40,401 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038640401 2024-11-19T17:50:40,406 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,406 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,411 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,411 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,411 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640393 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640401 2024-11-19T17:50:40,412 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:40,412 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038640401) roll requested 2024-11-19T17:50:40,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038640412 2024-11-19T17:50:40,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,428 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,428 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,428 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640401 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640412 2024-11-19T17:50:40,430 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:40,430 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,430 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,430 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038640412) roll requested 2024-11-19T17:50:40,430 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038640430 2024-11-19T17:50:40,439 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,439 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640412 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640430 2024-11-19T17:50:40,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:40,440 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038640430) roll requested 2024-11-19T17:50:40,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038640440 2024-11-19T17:50:40,447 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,447 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,447 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,447 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640430 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640440 2024-11-19T17:50:40,447 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:40,447 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,448 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,448 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038640440) roll requested 2024-11-19T17:50:40,448 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038640448 2024-11-19T17:50:40,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,474 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,474 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640440 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640448 2024-11-19T17:50:40,475 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:40,475 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,476 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038640448) roll requested 2024-11-19T17:50:40,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038640476 2024-11-19T17:50:40,493 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,493 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,494 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,494 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,494 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,494 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640448 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640476 2024-11-19T17:50:40,497 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:40,497 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038640476) roll requested 2024-11-19T17:50:40,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038640497 2024-11-19T17:50:40,515 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,515 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,515 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,515 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,515 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,515 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640476 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640497 2024-11-19T17:50:40,516 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:40,516 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,516 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,516 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038640497) roll requested 2024-11-19T17:50:40,517 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038640517 2024-11-19T17:50:40,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640497 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640517 2024-11-19T17:50:40,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:40,536 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,536 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,536 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038640517) roll requested 2024-11-19T17:50:40,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038640537 2024-11-19T17:50:40,569 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,569 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640517 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640537 2024-11-19T17:50:40,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:40,580 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038640537) roll requested 2024-11-19T17:50:40,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038640580 2024-11-19T17:50:40,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640537 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640580 2024-11-19T17:50:40,625 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:40,625 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038640580) roll requested 2024-11-19T17:50:40,626 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038640625 2024-11-19T17:50:40,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640580 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640625 2024-11-19T17:50:40,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:40,656 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,657 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,657 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038640625) roll requested 2024-11-19T17:50:40,657 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038640657 2024-11-19T17:50:40,665 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,666 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,668 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,668 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,668 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,668 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640625 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640657 2024-11-19T17:50:40,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:40,669 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,669 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038640657) roll requested 2024-11-19T17:50:40,669 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038640669 2024-11-19T17:50:40,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,697 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,697 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,697 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,697 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,697 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640657 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640669 2024-11-19T17:50:40,710 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:40,710 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,710 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,710 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038640669) roll requested 2024-11-19T17:50:40,710 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038640710 2024-11-19T17:50:40,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,728 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,728 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640669 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640710 2024-11-19T17:50:40,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:40,732 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,732 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,732 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038640710) roll requested 2024-11-19T17:50:40,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038640732 2024-11-19T17:50:40,748 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,748 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,748 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,748 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,748 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,748 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640710 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640732 2024-11-19T17:50:40,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:40,756 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038640732) roll requested 2024-11-19T17:50:40,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038640757 2024-11-19T17:50:40,775 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,775 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640732 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640757 2024-11-19T17:50:40,780 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:40,780 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,780 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,780 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038640757) roll requested 2024-11-19T17:50:40,781 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038640780 2024-11-19T17:50:40,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,794 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640757 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640780 2024-11-19T17:50:40,794 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:40,794 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,794 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,795 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038640780) roll requested 2024-11-19T17:50:40,795 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038640795 2024-11-19T17:50:40,806 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,807 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,807 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,807 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,807 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,808 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640780 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640795 2024-11-19T17:50:40,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:40,808 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,809 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038640795) roll requested 2024-11-19T17:50:40,809 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038640809 2024-11-19T17:50:40,824 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,824 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,824 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640795 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640809 2024-11-19T17:50:40,833 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:40,833 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,833 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,833 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038640809) roll requested 2024-11-19T17:50:40,833 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038640833 2024-11-19T17:50:40,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,849 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,849 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,849 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640809 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640833 2024-11-19T17:50:40,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:40,850 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,850 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,850 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038640833) roll requested 2024-11-19T17:50:40,850 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038640850 2024-11-19T17:50:40,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,867 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,867 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640833 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640850 2024-11-19T17:50:40,868 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:40,868 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038640850) roll requested 2024-11-19T17:50:40,869 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038640869 2024-11-19T17:50:40,884 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,884 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,885 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,885 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,885 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,885 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640850 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640869 2024-11-19T17:50:40,889 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:40,889 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,889 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,889 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038640869) roll requested 2024-11-19T17:50:40,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038640889 2024-11-19T17:50:40,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,908 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,908 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,908 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,908 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,908 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640869 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640889 2024-11-19T17:50:40,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:40,908 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038640889) roll requested 2024-11-19T17:50:40,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038640909 2024-11-19T17:50:40,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,919 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,919 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640889 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640909 2024-11-19T17:50:40,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:40,920 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038640909) roll requested 2024-11-19T17:50:40,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038640921 2024-11-19T17:50:40,927 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,927 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,927 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,928 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,928 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,928 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640909 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640921 2024-11-19T17:50:40,928 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:40,928 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,928 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038640921) roll requested 2024-11-19T17:50:40,929 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038640928 2024-11-19T17:50:40,936 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,937 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,937 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,937 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640921 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640928 2024-11-19T17:50:40,937 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:40,937 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,938 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,938 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038640928) roll requested 2024-11-19T17:50:40,938 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038640938 2024-11-19T17:50:40,948 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,948 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,948 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,948 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,948 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:40,948 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640928 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640938 2024-11-19T17:50:40,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:40,952 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:40,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038640938) roll requested 2024-11-19T17:50:40,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038640953 2024-11-19T17:50:41,006 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,006 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,006 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640938 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640953 2024-11-19T17:50:41,009 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,009 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038640953) roll requested 2024-11-19T17:50:41,010 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641009 2024-11-19T17:50:41,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,016 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,016 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,016 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,016 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038640953 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641009 2024-11-19T17:50:41,017 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,017 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641009) roll requested 2024-11-19T17:50:41,018 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641017 2024-11-19T17:50:41,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,025 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,025 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,025 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,025 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641009 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641017 2024-11-19T17:50:41,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,025 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,026 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641017) roll requested 2024-11-19T17:50:41,026 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641026 2024-11-19T17:50:41,034 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,034 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,034 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,034 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,034 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641017 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641026 2024-11-19T17:50:41,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,036 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,036 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,040 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641026) roll requested 2024-11-19T17:50:41,040 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641040 2024-11-19T17:50:41,046 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641026 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641040 2024-11-19T17:50:41,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,047 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641040) roll requested 2024-11-19T17:50:41,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641048 2024-11-19T17:50:41,059 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,059 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,059 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,059 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,059 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,059 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641040 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641048 2024-11-19T17:50:41,060 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,060 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,060 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,060 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641048) roll requested 2024-11-19T17:50:41,060 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641060 2024-11-19T17:50:41,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641048 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641060 2024-11-19T17:50:41,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,072 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,072 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,072 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641060) roll requested 2024-11-19T17:50:41,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641072 2024-11-19T17:50:41,080 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,080 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,080 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,080 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,080 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,081 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641060 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641072 2024-11-19T17:50:41,084 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,084 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,084 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641072) roll requested 2024-11-19T17:50:41,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641085 2024-11-19T17:50:41,098 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,098 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,098 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,098 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641072 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641085 2024-11-19T17:50:41,100 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,100 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,100 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,100 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641085) roll requested 2024-11-19T17:50:41,101 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641100 2024-11-19T17:50:41,117 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,117 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,117 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,117 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,117 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,117 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641085 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641100 2024-11-19T17:50:41,118 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,118 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,118 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,118 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641100) roll requested 2024-11-19T17:50:41,119 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641119 2024-11-19T17:50:41,132 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,132 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,133 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,133 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,133 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,133 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641100 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641119 2024-11-19T17:50:41,133 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,133 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,134 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,134 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641119) roll requested 2024-11-19T17:50:41,134 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641134 2024-11-19T17:50:41,144 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,144 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,144 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,145 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,145 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,145 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641119 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641134 2024-11-19T17:50:41,145 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,145 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641134) roll requested 2024-11-19T17:50:41,146 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641145 2024-11-19T17:50:41,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,163 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,163 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,163 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641134 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641145 2024-11-19T17:50:41,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,165 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641145) roll requested 2024-11-19T17:50:41,166 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641165 2024-11-19T17:50:41,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,175 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,175 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,175 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641145 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641165 2024-11-19T17:50:41,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,176 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,176 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641165) roll requested 2024-11-19T17:50:41,176 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641176 2024-11-19T17:50:41,191 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,191 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,191 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,192 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,192 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,192 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641165 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641176 2024-11-19T17:50:41,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,196 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,196 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641176) roll requested 2024-11-19T17:50:41,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641196 2024-11-19T17:50:41,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,202 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,202 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,202 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641176 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641196 2024-11-19T17:50:41,207 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,207 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,208 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641196) roll requested 2024-11-19T17:50:41,208 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641208 2024-11-19T17:50:41,213 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,213 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,213 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641196 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641208 2024-11-19T17:50:41,219 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,219 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641208) roll requested 2024-11-19T17:50:41,219 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641219 2024-11-19T17:50:41,225 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,225 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,225 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641208 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641219 2024-11-19T17:50:41,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,225 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641219) roll requested 2024-11-19T17:50:41,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641226 2024-11-19T17:50:41,248 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,248 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,249 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,249 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,249 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,249 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641219 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641226 2024-11-19T17:50:41,250 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,250 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,252 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641226) roll requested 2024-11-19T17:50:41,252 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641252 2024-11-19T17:50:41,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,263 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641226 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641252 2024-11-19T17:50:41,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,264 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641252) roll requested 2024-11-19T17:50:41,265 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641264 2024-11-19T17:50:41,274 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,275 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,275 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,275 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641252 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641264 2024-11-19T17:50:41,287 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,287 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,287 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641264) roll requested 2024-11-19T17:50:41,287 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641287 2024-11-19T17:50:41,294 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,294 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,294 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,295 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,295 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641264 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641287 2024-11-19T17:50:41,295 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,295 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,295 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,296 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641287) roll requested 2024-11-19T17:50:41,296 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641296 2024-11-19T17:50:41,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,312 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,312 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,313 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,313 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641287 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641296 2024-11-19T17:50:41,321 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,321 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,321 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,321 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641296) roll requested 2024-11-19T17:50:41,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641321 2024-11-19T17:50:41,330 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,330 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,331 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,331 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,331 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641296 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641321 2024-11-19T17:50:41,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,336 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641321) roll requested 2024-11-19T17:50:41,337 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641336 2024-11-19T17:50:41,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,347 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,347 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,347 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,347 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641321 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641336 2024-11-19T17:50:41,352 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,352 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,352 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,352 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641336) roll requested 2024-11-19T17:50:41,352 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641352 2024-11-19T17:50:41,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,363 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,363 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,363 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,363 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,363 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641336 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641352 2024-11-19T17:50:41,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,369 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641352) roll requested 2024-11-19T17:50:41,369 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641369 2024-11-19T17:50:41,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,379 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,379 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,379 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,379 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641352 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641369 2024-11-19T17:50:41,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,380 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641369) roll requested 2024-11-19T17:50:41,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641380 2024-11-19T17:50:41,389 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,390 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,390 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,390 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,390 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641369 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641380 2024-11-19T17:50:41,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,392 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,393 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,393 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641380) roll requested 2024-11-19T17:50:41,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641393 2024-11-19T17:50:41,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,407 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,408 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641380 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641393 2024-11-19T17:50:41,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,413 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641393) roll requested 2024-11-19T17:50:41,414 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641414 2024-11-19T17:50:41,420 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,420 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,420 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,420 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641393 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641414 2024-11-19T17:50:41,421 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,421 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,422 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,422 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641414) roll requested 2024-11-19T17:50:41,422 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641422 2024-11-19T17:50:41,434 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,434 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,434 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,434 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,434 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,434 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641414 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641422 2024-11-19T17:50:41,435 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,435 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,435 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,435 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641422) roll requested 2024-11-19T17:50:41,436 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641435 2024-11-19T17:50:41,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,446 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,446 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,447 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641422 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641435 2024-11-19T17:50:41,447 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,447 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641422 is not closed yet, will try archiving it next time 2024-11-19T17:50:41,447 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,447 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,447 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641435) roll requested 2024-11-19T17:50:41,448 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641447 2024-11-19T17:50:41,462 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,462 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641435 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641447 2024-11-19T17:50:41,482 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,482 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,482 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,482 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641447) roll requested 2024-11-19T17:50:41,482 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641482 2024-11-19T17:50:41,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,495 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641447 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641482 2024-11-19T17:50:41,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,500 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641482) roll requested 2024-11-19T17:50:41,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641501 2024-11-19T17:50:41,512 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,512 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,512 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641482 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641501 2024-11-19T17:50:41,517 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,517 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,517 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,517 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641501) roll requested 2024-11-19T17:50:41,517 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641517 2024-11-19T17:50:41,523 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,523 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,523 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641501 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641517 2024-11-19T17:50:41,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,524 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,524 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,524 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641517) roll requested 2024-11-19T17:50:41,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641524 2024-11-19T17:50:41,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,530 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,530 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,530 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641517 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641524 2024-11-19T17:50:41,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,539 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,539 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,539 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641524) roll requested 2024-11-19T17:50:41,539 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641539 2024-11-19T17:50:41,545 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,546 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641524 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641539 2024-11-19T17:50:41,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,548 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641539) roll requested 2024-11-19T17:50:41,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641549 2024-11-19T17:50:41,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641539 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641549 2024-11-19T17:50:41,560 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,560 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,561 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,561 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641549) roll requested 2024-11-19T17:50:41,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641561 2024-11-19T17:50:41,569 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641549 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641561 2024-11-19T17:50:41,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,572 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641561) roll requested 2024-11-19T17:50:41,573 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641572 2024-11-19T17:50:41,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,591 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641561 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641572 2024-11-19T17:50:41,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,593 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641572) roll requested 2024-11-19T17:50:41,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641593 2024-11-19T17:50:41,603 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,603 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,603 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,603 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,603 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,603 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641572 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641593 2024-11-19T17:50:41,614 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,614 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,614 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,614 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641593) roll requested 2024-11-19T17:50:41,615 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641614 2024-11-19T17:50:41,624 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,624 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,624 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,625 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,625 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,625 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641593 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641614 2024-11-19T17:50:41,628 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641614) roll requested 2024-11-19T17:50:41,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641628 2024-11-19T17:50:41,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,644 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,644 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,645 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641614 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641628 2024-11-19T17:50:41,649 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,649 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,649 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,649 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641628) roll requested 2024-11-19T17:50:41,649 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641649 2024-11-19T17:50:41,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641628 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641649 2024-11-19T17:50:41,663 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,663 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,663 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,663 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641649) roll requested 2024-11-19T17:50:41,663 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641663 2024-11-19T17:50:41,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,675 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,675 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,675 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,675 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641649 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641663 2024-11-19T17:50:41,686 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,686 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,686 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,686 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641663) roll requested 2024-11-19T17:50:41,686 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641686 2024-11-19T17:50:41,702 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,702 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,702 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,702 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,702 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,702 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641663 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641686 2024-11-19T17:50:41,713 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,713 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,713 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641686) roll requested 2024-11-19T17:50:41,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641714 2024-11-19T17:50:41,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,725 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,725 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641686 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641714 2024-11-19T17:50:41,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,736 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,737 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641714) roll requested 2024-11-19T17:50:41,737 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641737 2024-11-19T17:50:41,771 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,771 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,771 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,771 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641714 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641737 2024-11-19T17:50:41,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,772 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641737) roll requested 2024-11-19T17:50:41,772 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641772 2024-11-19T17:50:41,779 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,779 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,779 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,780 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,780 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,780 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641737 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641772 2024-11-19T17:50:41,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,789 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,789 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641772) roll requested 2024-11-19T17:50:41,789 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641789 2024-11-19T17:50:41,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,813 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,813 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,816 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,816 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641772 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641789 2024-11-19T17:50:41,824 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,824 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,824 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,825 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641789) roll requested 2024-11-19T17:50:41,825 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641825 2024-11-19T17:50:41,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,843 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,843 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,843 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,844 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,844 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641789 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641825 2024-11-19T17:50:41,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,848 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,848 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641825) roll requested 2024-11-19T17:50:41,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641849 2024-11-19T17:50:41,856 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,856 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,856 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,856 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,857 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,857 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641825 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641849 2024-11-19T17:50:41,857 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,857 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,858 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641849) roll requested 2024-11-19T17:50:41,858 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641858 2024-11-19T17:50:41,863 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,863 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,863 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641849 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641858 2024-11-19T17:50:41,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,864 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,864 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641858) roll requested 2024-11-19T17:50:41,864 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641864 2024-11-19T17:50:41,876 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,876 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,876 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641858 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641864 2024-11-19T17:50:41,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,880 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,880 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,880 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641864) roll requested 2024-11-19T17:50:41,880 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641880 2024-11-19T17:50:41,897 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,897 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,898 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,898 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,898 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,898 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641864 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641880 2024-11-19T17:50:41,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,900 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641880) roll requested 2024-11-19T17:50:41,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641901 2024-11-19T17:50:41,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,908 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,909 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,909 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641880 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641901 2024-11-19T17:50:41,909 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,909 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,909 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641901) roll requested 2024-11-19T17:50:41,910 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641910 2024-11-19T17:50:41,915 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,915 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,915 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,915 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,915 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,916 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641901 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641910 2024-11-19T17:50:41,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,917 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641910) roll requested 2024-11-19T17:50:41,917 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641917 2024-11-19T17:50:41,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,927 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,927 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,927 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,928 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,928 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641910 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641917 2024-11-19T17:50:41,928 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,928 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,928 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641917) roll requested 2024-11-19T17:50:41,929 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641929 2024-11-19T17:50:41,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,935 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,935 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,935 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,935 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,935 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641917 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641929 2024-11-19T17:50:41,942 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,942 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,943 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,943 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641929) roll requested 2024-11-19T17:50:41,943 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641943 2024-11-19T17:50:41,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,952 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,952 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,952 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,952 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,953 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641929 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641943 2024-11-19T17:50:41,961 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,961 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,961 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,961 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641943) roll requested 2024-11-19T17:50:41,961 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641961 2024-11-19T17:50:41,972 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,972 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,973 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,973 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,973 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,973 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641943 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641961 2024-11-19T17:50:41,976 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,976 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,976 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,976 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641961) roll requested 2024-11-19T17:50:41,976 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641976 2024-11-19T17:50:41,987 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,987 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,987 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,987 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,987 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,987 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641961 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641976 2024-11-19T17:50:41,988 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:41,988 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,988 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:41,988 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641976) roll requested 2024-11-19T17:50:41,988 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038641988 2024-11-19T17:50:41,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,995 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:41,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641976 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641988 2024-11-19T17:50:42,000 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,000 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,001 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038641988) roll requested 2024-11-19T17:50:42,001 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642001 2024-11-19T17:50:42,008 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,008 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,008 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,008 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,009 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038641988 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642001 2024-11-19T17:50:42,010 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,010 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,010 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,010 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642001) roll requested 2024-11-19T17:50:42,010 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642010 2024-11-19T17:50:42,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,020 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,020 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,020 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,020 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,020 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642001 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642010 2024-11-19T17:50:42,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,020 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,021 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,021 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642010) roll requested 2024-11-19T17:50:42,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642021 2024-11-19T17:50:42,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,027 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,027 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,027 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,027 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,027 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642010 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642021 2024-11-19T17:50:42,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,032 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642021) roll requested 2024-11-19T17:50:42,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642032 2024-11-19T17:50:42,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,041 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,041 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642021 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642032 2024-11-19T17:50:42,042 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,042 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,042 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,043 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642032) roll requested 2024-11-19T17:50:42,043 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642043 2024-11-19T17:50:42,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,056 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,056 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,056 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642032 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642043 2024-11-19T17:50:42,060 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,060 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,060 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,060 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642043) roll requested 2024-11-19T17:50:42,061 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642060 2024-11-19T17:50:42,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,071 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,071 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,071 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,071 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642043 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642060 2024-11-19T17:50:42,071 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,071 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,072 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,072 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642060) roll requested 2024-11-19T17:50:42,072 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642072 2024-11-19T17:50:42,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,085 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,085 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,085 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642060 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642072 2024-11-19T17:50:42,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,090 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,090 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,090 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642072) roll requested 2024-11-19T17:50:42,090 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642090 2024-11-19T17:50:42,106 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,106 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,106 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,106 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,106 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642072 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642090 2024-11-19T17:50:42,108 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,108 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,108 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,108 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642090) roll requested 2024-11-19T17:50:42,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642108 2024-11-19T17:50:42,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642090 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642108 2024-11-19T17:50:42,124 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,124 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,124 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,124 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642108) roll requested 2024-11-19T17:50:42,125 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642124 2024-11-19T17:50:42,130 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,130 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,130 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,130 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,130 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,131 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642108 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642124 2024-11-19T17:50:42,133 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,133 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,133 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642124) roll requested 2024-11-19T17:50:42,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642133 2024-11-19T17:50:42,145 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,145 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,145 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,145 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,145 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,145 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642124 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642133 2024-11-19T17:50:42,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,148 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642133) roll requested 2024-11-19T17:50:42,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642149 2024-11-19T17:50:42,154 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,154 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,154 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,154 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,154 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,154 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642133 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642149 2024-11-19T17:50:42,156 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,156 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,157 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642149) roll requested 2024-11-19T17:50:42,157 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642157 2024-11-19T17:50:42,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,176 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,176 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,176 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,176 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642149 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642157 2024-11-19T17:50:42,177 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,177 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642157) roll requested 2024-11-19T17:50:42,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642177 2024-11-19T17:50:42,190 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,190 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,191 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,191 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,192 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,192 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642157 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642177 2024-11-19T17:50:42,193 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,193 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,194 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,194 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642177) roll requested 2024-11-19T17:50:42,194 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642194 2024-11-19T17:50:42,212 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,212 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,213 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642177 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642194 2024-11-19T17:50:42,213 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,213 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,213 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,214 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642194) roll requested 2024-11-19T17:50:42,214 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642214 2024-11-19T17:50:42,223 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,223 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,223 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,223 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,223 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,223 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642194 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642214 2024-11-19T17:50:42,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,224 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,224 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,224 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642214) roll requested 2024-11-19T17:50:42,224 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642224 2024-11-19T17:50:42,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,230 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,230 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,230 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,230 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642214 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642224 2024-11-19T17:50:42,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,233 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642224) roll requested 2024-11-19T17:50:42,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642233 2024-11-19T17:50:42,248 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,249 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,249 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,249 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,249 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642224 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642233 2024-11-19T17:50:42,260 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,260 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,260 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642233) roll requested 2024-11-19T17:50:42,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642261 2024-11-19T17:50:42,272 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,272 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,273 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,273 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,273 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,273 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642233 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642261 2024-11-19T17:50:42,280 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,280 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,280 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,280 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642261) roll requested 2024-11-19T17:50:42,280 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642280 2024-11-19T17:50:42,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642261 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642280 2024-11-19T17:50:42,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,300 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642280) roll requested 2024-11-19T17:50:42,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642301 2024-11-19T17:50:42,309 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,310 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,310 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,310 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,310 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,310 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642280 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642301 2024-11-19T17:50:42,315 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,315 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,315 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,315 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642301) roll requested 2024-11-19T17:50:42,315 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642315 2024-11-19T17:50:42,326 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,326 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,327 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,327 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,327 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,327 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642301 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642315 2024-11-19T17:50:42,327 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,327 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,327 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642315) roll requested 2024-11-19T17:50:42,328 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642328 2024-11-19T17:50:42,335 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,335 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,335 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,335 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,335 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,335 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642315 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642328 2024-11-19T17:50:42,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,336 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642328) roll requested 2024-11-19T17:50:42,336 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642336 2024-11-19T17:50:42,343 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642328 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642336 2024-11-19T17:50:42,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,344 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,345 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642336) roll requested 2024-11-19T17:50:42,345 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642345 2024-11-19T17:50:42,361 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,362 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642336 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642345 2024-11-19T17:50:42,366 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,366 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,366 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,366 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642345) roll requested 2024-11-19T17:50:42,366 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642366 2024-11-19T17:50:42,379 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,379 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,379 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,379 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,380 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,380 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642345 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642366 2024-11-19T17:50:42,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,389 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642366) roll requested 2024-11-19T17:50:42,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642389 2024-11-19T17:50:42,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,400 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,400 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,400 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,400 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,400 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642366 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642389 2024-11-19T17:50:42,406 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,406 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642389) roll requested 2024-11-19T17:50:42,407 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642407 2024-11-19T17:50:42,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,414 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642389 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642407 2024-11-19T17:50:42,417 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,417 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,417 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,417 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642407) roll requested 2024-11-19T17:50:42,417 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642417 2024-11-19T17:50:42,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642407 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642417 2024-11-19T17:50:42,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,432 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,432 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642417) roll requested 2024-11-19T17:50:42,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642432 2024-11-19T17:50:42,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,441 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,441 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642417 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642432 2024-11-19T17:50:42,445 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,445 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,445 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642432) roll requested 2024-11-19T17:50:42,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642445 2024-11-19T17:50:42,454 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,454 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,454 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,455 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,455 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642432 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642445 2024-11-19T17:50:42,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,460 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,460 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642445) roll requested 2024-11-19T17:50:42,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642460 2024-11-19T17:50:42,472 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,472 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,472 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642445 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642460 2024-11-19T17:50:42,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,484 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642460) roll requested 2024-11-19T17:50:42,484 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642484 2024-11-19T17:50:42,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,495 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642460 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642484 2024-11-19T17:50:42,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,496 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642484) roll requested 2024-11-19T17:50:42,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642496 2024-11-19T17:50:42,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,508 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,508 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,508 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,508 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,508 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642484 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642496 2024-11-19T17:50:42,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,512 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,513 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642496) roll requested 2024-11-19T17:50:42,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642513 2024-11-19T17:50:42,523 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,523 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,523 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642496 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642513 2024-11-19T17:50:42,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,528 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642513) roll requested 2024-11-19T17:50:42,529 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642528 2024-11-19T17:50:42,535 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,535 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,535 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,535 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,535 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,535 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642513 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642528 2024-11-19T17:50:42,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,536 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,536 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,536 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642528) roll requested 2024-11-19T17:50:42,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642536 2024-11-19T17:50:42,544 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,544 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,545 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,545 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,545 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642528 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642536 2024-11-19T17:50:42,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,548 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642536) roll requested 2024-11-19T17:50:42,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642548 2024-11-19T17:50:42,557 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,557 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,557 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,557 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,557 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,557 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642536 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642548 2024-11-19T17:50:42,560 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,560 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,560 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,560 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642548) roll requested 2024-11-19T17:50:42,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642560 2024-11-19T17:50:42,566 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,567 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,567 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,567 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,567 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,567 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642548 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642560 2024-11-19T17:50:42,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,568 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,569 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642560) roll requested 2024-11-19T17:50:42,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642569 2024-11-19T17:50:42,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,578 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,578 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,578 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,578 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642560 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642569 2024-11-19T17:50:42,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,580 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642569) roll requested 2024-11-19T17:50:42,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642580 2024-11-19T17:50:42,588 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,589 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,589 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,589 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642569 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642580 2024-11-19T17:50:42,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,592 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642580) roll requested 2024-11-19T17:50:42,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642593 2024-11-19T17:50:42,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642580 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642593 2024-11-19T17:50:42,609 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,609 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,609 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642593) roll requested 2024-11-19T17:50:42,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642609 2024-11-19T17:50:42,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642593 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642609 2024-11-19T17:50:42,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,624 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642609) roll requested 2024-11-19T17:50:42,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642625 2024-11-19T17:50:42,631 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,631 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,632 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642609 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642625 2024-11-19T17:50:42,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,632 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,633 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642625) roll requested 2024-11-19T17:50:42,633 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642633 2024-11-19T17:50:42,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,642 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,643 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,644 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642625 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642633 2024-11-19T17:50:42,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,644 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,644 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642633) roll requested 2024-11-19T17:50:42,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642645 2024-11-19T17:50:42,664 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,664 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,665 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,665 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,665 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,665 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642633 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642645 2024-11-19T17:50:42,677 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,677 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,677 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,677 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642645) roll requested 2024-11-19T17:50:42,677 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642677 2024-11-19T17:50:42,702 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,702 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,702 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,702 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,702 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642645 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642677 2024-11-19T17:50:42,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,708 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642677) roll requested 2024-11-19T17:50:42,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642708 2024-11-19T17:50:42,716 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,716 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,716 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,716 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,717 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,717 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642677 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642708 2024-11-19T17:50:42,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,724 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,724 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642708) roll requested 2024-11-19T17:50:42,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642725 2024-11-19T17:50:42,730 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,730 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642708 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642725 2024-11-19T17:50:42,731 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,731 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,731 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,731 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642725) roll requested 2024-11-19T17:50:42,732 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642731 2024-11-19T17:50:42,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,745 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642725 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642731 2024-11-19T17:50:42,755 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,755 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,755 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,755 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642731) roll requested 2024-11-19T17:50:42,756 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642755 2024-11-19T17:50:42,771 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,771 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,771 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,771 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642731 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642755 2024-11-19T17:50:42,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,772 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642755) roll requested 2024-11-19T17:50:42,772 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642772 2024-11-19T17:50:42,785 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,785 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,785 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,785 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,785 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,786 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642755 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642772 2024-11-19T17:50:42,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,788 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642772) roll requested 2024-11-19T17:50:42,789 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642789 2024-11-19T17:50:42,800 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642772 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642789 2024-11-19T17:50:42,803 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,803 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,803 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,803 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642789) roll requested 2024-11-19T17:50:42,803 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642803 2024-11-19T17:50:42,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642789 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642803 2024-11-19T17:50:42,817 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,817 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,817 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,817 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642803) roll requested 2024-11-19T17:50:42,817 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642817 2024-11-19T17:50:42,823 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,823 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,823 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,823 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,824 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,824 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642803 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642817 2024-11-19T17:50:42,824 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,824 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,824 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,825 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642817) roll requested 2024-11-19T17:50:42,825 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642825 2024-11-19T17:50:42,830 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,830 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,830 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,830 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,830 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642817 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642825 2024-11-19T17:50:42,832 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,832 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,832 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,832 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642825) roll requested 2024-11-19T17:50:42,833 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642832 2024-11-19T17:50:42,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642825 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642832 2024-11-19T17:50:42,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,840 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642832) roll requested 2024-11-19T17:50:42,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642840 2024-11-19T17:50:42,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642832 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642840 2024-11-19T17:50:42,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,848 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,848 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642840) roll requested 2024-11-19T17:50:42,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642848 2024-11-19T17:50:42,853 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,854 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,854 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,854 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642840 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642848 2024-11-19T17:50:42,854 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,854 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,854 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,854 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642848) roll requested 2024-11-19T17:50:42,855 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642854 2024-11-19T17:50:42,862 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,862 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,862 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,862 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,862 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,863 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642848 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642854 2024-11-19T17:50:42,863 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,863 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,864 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642854) roll requested 2024-11-19T17:50:42,864 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642864 2024-11-19T17:50:42,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,872 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,873 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642854 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642864 2024-11-19T17:50:42,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,873 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,873 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642864) roll requested 2024-11-19T17:50:42,873 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642873 2024-11-19T17:50:42,879 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642864 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642873 2024-11-19T17:50:42,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,881 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642873) roll requested 2024-11-19T17:50:42,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642881 2024-11-19T17:50:42,888 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,889 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,889 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642873 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642881 2024-11-19T17:50:42,897 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,897 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,897 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,897 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642881) roll requested 2024-11-19T17:50:42,897 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642897 2024-11-19T17:50:42,903 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,903 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,903 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,903 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,903 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,903 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642881 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642897 2024-11-19T17:50:42,904 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,904 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,904 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,905 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642897) roll requested 2024-11-19T17:50:42,905 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642905 2024-11-19T17:50:42,912 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,912 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,912 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,913 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,913 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,913 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642897 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642905 2024-11-19T17:50:42,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,920 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642905) roll requested 2024-11-19T17:50:42,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642920 2024-11-19T17:50:42,925 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,925 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,925 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,926 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642905 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642920 2024-11-19T17:50:42,926 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,926 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,926 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,926 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642920) roll requested 2024-11-19T17:50:42,926 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642926 2024-11-19T17:50:42,933 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,933 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,933 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,933 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,933 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,933 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642920 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642926 2024-11-19T17:50:42,944 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,944 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,944 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642926) roll requested 2024-11-19T17:50:42,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642944 2024-11-19T17:50:42,950 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,950 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,950 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,950 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642926 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642944 2024-11-19T17:50:42,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,952 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642944) roll requested 2024-11-19T17:50:42,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642952 2024-11-19T17:50:42,961 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,961 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,961 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,961 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,962 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,962 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642944 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642952 2024-11-19T17:50:42,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,968 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,969 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642952) roll requested 2024-11-19T17:50:42,969 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642969 2024-11-19T17:50:42,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,979 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,980 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,980 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,980 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642952 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642969 2024-11-19T17:50:42,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,980 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642969) roll requested 2024-11-19T17:50:42,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642980 2024-11-19T17:50:42,988 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,988 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,989 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,989 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:42,989 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642969 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642980 2024-11-19T17:50:42,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:42,997 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:42,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642980) roll requested 2024-11-19T17:50:42,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038642997 2024-11-19T17:50:43,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,002 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,002 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,002 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,002 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,002 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642980 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642997 2024-11-19T17:50:43,004 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,004 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038642997) roll requested 2024-11-19T17:50:43,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643004 2024-11-19T17:50:43,012 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,012 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,012 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,013 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,013 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,013 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038642997 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643004 2024-11-19T17:50:43,014 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,013 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643004) roll requested 2024-11-19T17:50:43,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643014 2024-11-19T17:50:43,029 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,029 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,029 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,029 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,029 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643004 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643014 2024-11-19T17:50:43,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,032 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643014) roll requested 2024-11-19T17:50:43,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643033 2024-11-19T17:50:43,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,041 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,041 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,041 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643014 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643033 2024-11-19T17:50:43,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,048 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643033) roll requested 2024-11-19T17:50:43,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643049 2024-11-19T17:50:43,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,059 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,059 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643033 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643049 2024-11-19T17:50:43,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,059 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,059 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,059 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643049) roll requested 2024-11-19T17:50:43,059 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643059 2024-11-19T17:50:43,071 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,071 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,071 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,071 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,071 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,071 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643049 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643059 2024-11-19T17:50:43,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,072 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,072 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,072 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643059) roll requested 2024-11-19T17:50:43,072 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643072 2024-11-19T17:50:43,079 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,080 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,080 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,080 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,080 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,080 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643059 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643072 2024-11-19T17:50:43,081 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,081 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,082 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,082 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643072) roll requested 2024-11-19T17:50:43,082 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643082 2024-11-19T17:50:43,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,088 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,088 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643072 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643082 2024-11-19T17:50:43,093 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,093 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,093 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643082) roll requested 2024-11-19T17:50:43,094 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643094 2024-11-19T17:50:43,101 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,101 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,101 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,101 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,104 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643082 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643094 2024-11-19T17:50:43,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,105 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643094) roll requested 2024-11-19T17:50:43,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643106 2024-11-19T17:50:43,121 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,123 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643094 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643106 2024-11-19T17:50:43,123 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,124 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643094 is not closed yet, will try archiving it next time 2024-11-19T17:50:43,123 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,124 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,124 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643106) roll requested 2024-11-19T17:50:43,124 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643124 2024-11-19T17:50:43,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,151 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643106 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643124 2024-11-19T17:50:43,156 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,156 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,156 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643124) roll requested 2024-11-19T17:50:43,157 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643156 2024-11-19T17:50:43,168 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,168 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,168 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,168 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,168 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,168 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643124 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643156 2024-11-19T17:50:43,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,172 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643156) roll requested 2024-11-19T17:50:43,172 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643172 2024-11-19T17:50:43,180 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,180 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,180 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,181 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,181 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,181 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643156 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643172 2024-11-19T17:50:43,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,184 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643172) roll requested 2024-11-19T17:50:43,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643185 2024-11-19T17:50:43,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643172 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643185 2024-11-19T17:50:43,200 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,200 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643185) roll requested 2024-11-19T17:50:43,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643201 2024-11-19T17:50:43,207 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,207 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,207 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,207 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,207 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643185 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643201 2024-11-19T17:50:43,209 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,209 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,209 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,209 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643201) roll requested 2024-11-19T17:50:43,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643209 2024-11-19T17:50:43,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643201 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643209 2024-11-19T17:50:43,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,216 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643209) roll requested 2024-11-19T17:50:43,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643216 2024-11-19T17:50:43,227 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,227 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,227 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,227 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643209 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643216 2024-11-19T17:50:43,227 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,227 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,227 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,228 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643216) roll requested 2024-11-19T17:50:43,228 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643228 2024-11-19T17:50:43,233 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,234 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,234 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,234 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,234 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,234 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643216 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643228 2024-11-19T17:50:43,235 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,235 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,235 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,235 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643228) roll requested 2024-11-19T17:50:43,235 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643235 2024-11-19T17:50:43,244 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643228 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643235 2024-11-19T17:50:43,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,252 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,253 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643235) roll requested 2024-11-19T17:50:43,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643253 2024-11-19T17:50:43,259 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,259 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,259 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,259 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,259 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,260 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643235 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643253 2024-11-19T17:50:43,261 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,261 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643253) roll requested 2024-11-19T17:50:43,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643261 2024-11-19T17:50:43,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643253 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643261 2024-11-19T17:50:43,282 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,282 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,282 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,282 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643261) roll requested 2024-11-19T17:50:43,282 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643282 2024-11-19T17:50:43,295 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,295 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,295 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,295 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643261 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643282 2024-11-19T17:50:43,296 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,296 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,296 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,296 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643282) roll requested 2024-11-19T17:50:43,296 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643296 2024-11-19T17:50:43,301 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,301 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,302 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,302 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,302 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,302 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643282 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643296 2024-11-19T17:50:43,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,304 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643296) roll requested 2024-11-19T17:50:43,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643305 2024-11-19T17:50:43,314 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,315 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,315 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,315 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643296 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643305 2024-11-19T17:50:43,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,316 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,316 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,316 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643305) roll requested 2024-11-19T17:50:43,316 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643316 2024-11-19T17:50:43,328 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,329 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,329 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,329 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,329 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643305 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643316 2024-11-19T17:50:43,330 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,333 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,333 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,333 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643316) roll requested 2024-11-19T17:50:43,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643333 2024-11-19T17:50:43,350 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,350 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,350 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,350 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,351 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,352 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643316 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643333 2024-11-19T17:50:43,352 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,352 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,352 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,352 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643333) roll requested 2024-11-19T17:50:43,352 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643352 2024-11-19T17:50:43,358 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,358 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,359 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,359 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,359 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,359 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643333 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643352 2024-11-19T17:50:43,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,360 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,360 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,360 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643352) roll requested 2024-11-19T17:50:43,360 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643360 2024-11-19T17:50:43,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643352 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643360 2024-11-19T17:50:43,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,368 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643360) roll requested 2024-11-19T17:50:43,369 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643369 2024-11-19T17:50:43,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643360 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643369 2024-11-19T17:50:43,378 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,378 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,378 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,378 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643369) roll requested 2024-11-19T17:50:43,379 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643378 2024-11-19T17:50:43,386 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,386 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,386 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,386 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643369 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643378 2024-11-19T17:50:43,388 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,388 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,388 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,388 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643378) roll requested 2024-11-19T17:50:43,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643388 2024-11-19T17:50:43,406 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,406 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,407 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,407 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643378 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643388 2024-11-19T17:50:43,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,408 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,408 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643388) roll requested 2024-11-19T17:50:43,409 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643408 2024-11-19T17:50:43,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643388 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643408 2024-11-19T17:50:43,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,428 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643408) roll requested 2024-11-19T17:50:43,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643428 2024-11-19T17:50:43,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643408 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643428 2024-11-19T17:50:43,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,449 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,449 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643428) roll requested 2024-11-19T17:50:43,449 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643449 2024-11-19T17:50:43,462 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,462 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643428 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643449 2024-11-19T17:50:43,471 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,471 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,471 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,471 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643449) roll requested 2024-11-19T17:50:43,471 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643471 2024-11-19T17:50:43,478 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,478 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,478 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,478 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,478 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,478 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643449 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643471 2024-11-19T17:50:43,479 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,480 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,480 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643471) roll requested 2024-11-19T17:50:43,480 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643480 2024-11-19T17:50:43,485 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,486 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643471 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643480 2024-11-19T17:50:43,488 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,488 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,488 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,488 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643480) roll requested 2024-11-19T17:50:43,489 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643488 2024-11-19T17:50:43,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,501 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643480 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643488 2024-11-19T17:50:43,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,501 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,502 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643488) roll requested 2024-11-19T17:50:43,502 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643502 2024-11-19T17:50:43,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,512 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,512 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,512 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,512 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,512 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643488 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643502 2024-11-19T17:50:43,513 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,513 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,513 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,513 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643502) roll requested 2024-11-19T17:50:43,514 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643514 2024-11-19T17:50:43,519 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,520 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,520 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,520 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,520 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643502 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643514 2024-11-19T17:50:43,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,524 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,524 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,524 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643514) roll requested 2024-11-19T17:50:43,524 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643524 2024-11-19T17:50:43,529 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,536 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,536 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,536 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,536 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643514 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643524 2024-11-19T17:50:43,537 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,537 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643524) roll requested 2024-11-19T17:50:43,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643537 2024-11-19T17:50:43,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,550 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,550 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,550 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643524 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643537 2024-11-19T17:50:43,551 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,551 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,555 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643537) roll requested 2024-11-19T17:50:43,555 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643555 2024-11-19T17:50:43,568 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,569 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,569 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,569 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,569 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,569 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643537 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643555 2024-11-19T17:50:43,570 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,570 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,570 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,570 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643555) roll requested 2024-11-19T17:50:43,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643570 2024-11-19T17:50:43,580 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,581 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,581 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,581 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,581 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643555 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643570 2024-11-19T17:50:43,582 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,582 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,582 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,582 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643570) roll requested 2024-11-19T17:50:43,582 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643582 2024-11-19T17:50:43,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,592 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643570 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643582 2024-11-19T17:50:43,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,593 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,594 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643582) roll requested 2024-11-19T17:50:43,594 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643594 2024-11-19T17:50:43,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,603 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,603 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,603 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,603 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,603 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643582 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643594 2024-11-19T17:50:43,603 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,603 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643594) roll requested 2024-11-19T17:50:43,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643604 2024-11-19T17:50:43,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,616 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,616 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,616 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643594 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643604 2024-11-19T17:50:43,618 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,618 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,618 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,618 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643604) roll requested 2024-11-19T17:50:43,618 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643618 2024-11-19T17:50:43,632 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,632 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,632 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,632 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,632 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,632 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643604 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643618 2024-11-19T17:50:43,633 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,633 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,633 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,633 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643618) roll requested 2024-11-19T17:50:43,633 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643633 2024-11-19T17:50:43,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,643 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,643 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,643 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,643 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643618 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643633 2024-11-19T17:50:43,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,644 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643633) roll requested 2024-11-19T17:50:43,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643645 2024-11-19T17:50:43,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643633 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643645 2024-11-19T17:50:43,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,660 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643645) roll requested 2024-11-19T17:50:43,660 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643660 2024-11-19T17:50:43,668 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,668 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,668 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,668 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,668 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,668 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643645 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643660 2024-11-19T17:50:43,670 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,670 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,670 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,670 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643660) roll requested 2024-11-19T17:50:43,670 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643670 2024-11-19T17:50:43,676 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,677 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,677 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,677 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643660 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643670 2024-11-19T17:50:43,679 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,679 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,679 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,679 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643670) roll requested 2024-11-19T17:50:43,679 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643679 2024-11-19T17:50:43,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,695 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643670 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643679 2024-11-19T17:50:43,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,697 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643679) roll requested 2024-11-19T17:50:43,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643697 2024-11-19T17:50:43,706 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643679 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643697 2024-11-19T17:50:43,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,708 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643697) roll requested 2024-11-19T17:50:43,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643709 2024-11-19T17:50:43,719 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,719 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,719 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,719 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,719 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643697 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643709 2024-11-19T17:50:43,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,724 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,724 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643709) roll requested 2024-11-19T17:50:43,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643725 2024-11-19T17:50:43,735 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,735 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,735 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,736 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,736 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,736 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643709 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643725 2024-11-19T17:50:43,740 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,744 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643725) roll requested 2024-11-19T17:50:43,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643744 2024-11-19T17:50:43,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643725 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643744 2024-11-19T17:50:43,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,764 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643744) roll requested 2024-11-19T17:50:43,765 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643765 2024-11-19T17:50:43,776 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,777 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,777 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,777 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643744 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643765 2024-11-19T17:50:43,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,778 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,778 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643765) roll requested 2024-11-19T17:50:43,778 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643778 2024-11-19T17:50:43,796 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,796 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,796 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,797 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,797 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,797 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643765 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643778 2024-11-19T17:50:43,798 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,798 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,798 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,798 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643778) roll requested 2024-11-19T17:50:43,798 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643798 2024-11-19T17:50:43,815 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,815 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,815 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,815 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,815 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,815 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643778 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643798 2024-11-19T17:50:43,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,820 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,820 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643798) roll requested 2024-11-19T17:50:43,820 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643820 2024-11-19T17:50:43,828 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,828 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,828 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,828 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,828 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,828 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643798 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643820 2024-11-19T17:50:43,829 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,829 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,829 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643820) roll requested 2024-11-19T17:50:43,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643829 2024-11-19T17:50:43,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,836 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,837 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,837 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643820 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643829 2024-11-19T17:50:43,846 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,847 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643829) roll requested 2024-11-19T17:50:43,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643847 2024-11-19T17:50:43,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,856 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,856 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,856 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,856 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,856 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643829 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643847 2024-11-19T17:50:43,857 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,857 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643847) roll requested 2024-11-19T17:50:43,857 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643857 2024-11-19T17:50:43,867 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,867 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,867 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,867 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643847 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643857 2024-11-19T17:50:43,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,868 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,868 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643857) roll requested 2024-11-19T17:50:43,868 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643868 2024-11-19T17:50:43,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,874 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,874 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,874 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,874 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,874 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643857 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643868 2024-11-19T17:50:43,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,876 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643868) roll requested 2024-11-19T17:50:43,877 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643877 2024-11-19T17:50:43,883 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,883 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,884 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,884 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,884 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,884 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643868 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643877 2024-11-19T17:50:43,892 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,893 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643877) roll requested 2024-11-19T17:50:43,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643893 2024-11-19T17:50:43,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,918 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643877 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643893 2024-11-19T17:50:43,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,921 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643893) roll requested 2024-11-19T17:50:43,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643921 2024-11-19T17:50:43,935 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,935 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,935 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,935 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,936 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,936 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643893 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643921 2024-11-19T17:50:43,936 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,936 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,936 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643921) roll requested 2024-11-19T17:50:43,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643937 2024-11-19T17:50:43,972 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,972 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,972 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,975 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,975 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:43,975 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643921 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643937 2024-11-19T17:50:43,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:43,993 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:43,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643937) roll requested 2024-11-19T17:50:43,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038643993 2024-11-19T17:50:44,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,032 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,032 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643937 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643993 2024-11-19T17:50:44,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,036 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,036 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,036 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038643993) roll requested 2024-11-19T17:50:44,036 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644036 2024-11-19T17:50:44,048 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,048 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,048 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,048 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,048 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,048 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038643993 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644036 2024-11-19T17:50:44,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,052 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,053 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,053 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644036) roll requested 2024-11-19T17:50:44,053 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644053 2024-11-19T17:50:44,066 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,066 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,066 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,066 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,066 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,066 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644036 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644053 2024-11-19T17:50:44,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,068 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,068 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,068 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644053) roll requested 2024-11-19T17:50:44,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644068 2024-11-19T17:50:44,074 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,074 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,074 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,074 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,074 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,074 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644053 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644068 2024-11-19T17:50:44,075 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,075 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,075 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,075 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644068) roll requested 2024-11-19T17:50:44,075 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644075 2024-11-19T17:50:44,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,085 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,085 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,085 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644068 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644075 2024-11-19T17:50:44,086 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,086 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,086 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,086 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644075) roll requested 2024-11-19T17:50:44,086 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644086 2024-11-19T17:50:44,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,106 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,106 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,106 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644075 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644086 2024-11-19T17:50:44,108 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,108 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,108 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,108 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644086) roll requested 2024-11-19T17:50:44,108 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644108 2024-11-19T17:50:44,116 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,116 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,116 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,117 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,117 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,117 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644086 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644108 2024-11-19T17:50:44,127 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,127 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,127 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,127 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644108) roll requested 2024-11-19T17:50:44,128 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644127 2024-11-19T17:50:44,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,136 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,137 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,137 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,137 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644108 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644127 2024-11-19T17:50:44,141 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,141 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,141 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,141 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644127) roll requested 2024-11-19T17:50:44,141 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644141 2024-11-19T17:50:44,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644127 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644141 2024-11-19T17:50:44,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,152 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,152 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644141) roll requested 2024-11-19T17:50:44,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644153 2024-11-19T17:50:44,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,165 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,165 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644141 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644153 2024-11-19T17:50:44,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,168 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,169 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,169 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644153) roll requested 2024-11-19T17:50:44,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644169 2024-11-19T17:50:44,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,176 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,176 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,176 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,176 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644153 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644169 2024-11-19T17:50:44,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,176 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644169) roll requested 2024-11-19T17:50:44,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644177 2024-11-19T17:50:44,185 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,185 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,185 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,185 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,185 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,186 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644169 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644177 2024-11-19T17:50:44,187 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,186 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,187 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,187 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644177) roll requested 2024-11-19T17:50:44,187 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644187 2024-11-19T17:50:44,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,194 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644177 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644187 2024-11-19T17:50:44,200 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,200 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,200 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,200 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644187) roll requested 2024-11-19T17:50:44,200 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644200 2024-11-19T17:50:44,233 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,233 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,233 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644187 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644200 2024-11-19T17:50:44,248 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,248 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,248 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,248 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644200) roll requested 2024-11-19T17:50:44,248 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644248 2024-11-19T17:50:44,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644200 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644248 2024-11-19T17:50:44,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,256 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,256 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,256 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644248) roll requested 2024-11-19T17:50:44,256 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644256 2024-11-19T17:50:44,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644248 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644256 2024-11-19T17:50:44,263 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,263 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,263 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,263 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644256) roll requested 2024-11-19T17:50:44,264 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644263 2024-11-19T17:50:44,272 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,272 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,276 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,276 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,276 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644256 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644263 2024-11-19T17:50:44,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,277 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644263) roll requested 2024-11-19T17:50:44,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644277 2024-11-19T17:50:44,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,285 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,285 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,285 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,285 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644263 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644277 2024-11-19T17:50:44,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,288 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,288 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644277) roll requested 2024-11-19T17:50:44,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644288 2024-11-19T17:50:44,295 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,295 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,295 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,295 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644277 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644288 2024-11-19T17:50:44,297 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,296 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,297 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,297 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644288) roll requested 2024-11-19T17:50:44,297 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644297 2024-11-19T17:50:44,303 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,303 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,303 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,303 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,304 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644288 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644297 2024-11-19T17:50:44,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,304 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644297) roll requested 2024-11-19T17:50:44,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644305 2024-11-19T17:50:44,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644297 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644305 2024-11-19T17:50:44,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,312 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644305) roll requested 2024-11-19T17:50:44,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644312 2024-11-19T17:50:44,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,318 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,318 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,318 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,318 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644305 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644312 2024-11-19T17:50:44,321 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,324 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644312) roll requested 2024-11-19T17:50:44,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644325 2024-11-19T17:50:44,330 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,336 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,336 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,339 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,339 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,339 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644312 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644325 2024-11-19T17:50:44,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,344 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,344 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644325) roll requested 2024-11-19T17:50:44,345 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644344 2024-11-19T17:50:44,359 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,359 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,359 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,359 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,359 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,359 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644325 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644344 2024-11-19T17:50:44,361 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,364 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,364 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,364 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644344) roll requested 2024-11-19T17:50:44,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644364 2024-11-19T17:50:44,370 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,371 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644344 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644364 2024-11-19T17:50:44,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,372 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,372 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644364) roll requested 2024-11-19T17:50:44,372 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644372 2024-11-19T17:50:44,387 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,387 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,387 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,387 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,387 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,387 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644364 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644372 2024-11-19T17:50:44,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,393 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,393 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,393 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644372) roll requested 2024-11-19T17:50:44,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644393 2024-11-19T17:50:44,404 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,404 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,404 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,405 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644372 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644393 2024-11-19T17:50:44,409 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,409 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,409 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,409 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644393) roll requested 2024-11-19T17:50:44,410 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644409 2024-11-19T17:50:44,422 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,422 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644393 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644409 2024-11-19T17:50:44,429 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,429 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,429 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,429 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644409) roll requested 2024-11-19T17:50:44,429 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644429 2024-11-19T17:50:44,445 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,445 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644409 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644429 2024-11-19T17:50:44,446 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,446 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,446 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,446 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644429) roll requested 2024-11-19T17:50:44,446 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644446 2024-11-19T17:50:44,452 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,452 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,452 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,452 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,452 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,453 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644429 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644446 2024-11-19T17:50:44,458 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,458 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,459 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,459 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644446) roll requested 2024-11-19T17:50:44,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644459 2024-11-19T17:50:44,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,474 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,474 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,474 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,474 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644446 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644459 2024-11-19T17:50:44,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,476 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,476 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644459) roll requested 2024-11-19T17:50:44,477 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644476 2024-11-19T17:50:44,481 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,482 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,482 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644459 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644476 2024-11-19T17:50:44,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,484 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644476) roll requested 2024-11-19T17:50:44,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644485 2024-11-19T17:50:44,492 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,492 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,492 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,492 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,492 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644476 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644485 2024-11-19T17:50:44,497 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,497 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644485) roll requested 2024-11-19T17:50:44,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644497 2024-11-19T17:50:44,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,502 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,502 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,502 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644485 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644497 2024-11-19T17:50:44,502 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,502 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,502 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644497) roll requested 2024-11-19T17:50:44,503 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644503 2024-11-19T17:50:44,515 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,515 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,515 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,515 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,515 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,515 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644497 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644503 2024-11-19T17:50:44,515 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,515 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,516 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644503) roll requested 2024-11-19T17:50:44,516 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644516 2024-11-19T17:50:44,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644503 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644516 2024-11-19T17:50:44,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,528 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644516) roll requested 2024-11-19T17:50:44,529 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644528 2024-11-19T17:50:44,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644516 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644528 2024-11-19T17:50:44,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,544 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,544 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644528) roll requested 2024-11-19T17:50:44,545 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644544 2024-11-19T17:50:44,552 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,552 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,552 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,552 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,553 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,553 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644528 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644544 2024-11-19T17:50:44,561 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,561 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,561 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,561 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644544) roll requested 2024-11-19T17:50:44,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644561 2024-11-19T17:50:44,569 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,569 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,569 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,569 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,572 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,572 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644544 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644561 2024-11-19T17:50:44,573 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,573 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,573 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,573 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644561) roll requested 2024-11-19T17:50:44,573 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644573 2024-11-19T17:50:44,588 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,589 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,589 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644561 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644573 2024-11-19T17:50:44,590 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,590 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,590 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,590 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644573) roll requested 2024-11-19T17:50:44,590 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644590 2024-11-19T17:50:44,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,617 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644573 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644590 2024-11-19T17:50:44,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,620 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,621 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644590) roll requested 2024-11-19T17:50:44,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644621 2024-11-19T17:50:44,628 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,628 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,629 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,629 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,630 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,630 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644590 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644621 2024-11-19T17:50:44,630 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,630 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,631 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,631 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644621) roll requested 2024-11-19T17:50:44,631 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644631 2024-11-19T17:50:44,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644621 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644631 2024-11-19T17:50:44,653 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,653 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,653 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,653 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644631) roll requested 2024-11-19T17:50:44,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644654 2024-11-19T17:50:44,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,667 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,667 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,667 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644631 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644654 2024-11-19T17:50:44,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,667 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,668 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,668 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644654) roll requested 2024-11-19T17:50:44,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644668 2024-11-19T17:50:44,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,683 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,683 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644654 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644668 2024-11-19T17:50:44,683 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,683 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,683 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,683 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644668) roll requested 2024-11-19T17:50:44,683 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644683 2024-11-19T17:50:44,697 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,698 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,698 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,698 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,698 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,698 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644668 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644683 2024-11-19T17:50:44,698 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,698 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,698 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,698 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644683) roll requested 2024-11-19T17:50:44,699 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644699 2024-11-19T17:50:44,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,709 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,709 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644683 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644699 2024-11-19T17:50:44,712 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,712 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,712 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,712 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644699) roll requested 2024-11-19T17:50:44,713 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644712 2024-11-19T17:50:44,718 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,719 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,719 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,719 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,719 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644699 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644712 2024-11-19T17:50:44,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,724 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,724 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644712) roll requested 2024-11-19T17:50:44,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644724 2024-11-19T17:50:44,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644712 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644724 2024-11-19T17:50:44,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,736 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,737 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644724) roll requested 2024-11-19T17:50:44,737 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644737 2024-11-19T17:50:44,749 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,749 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,750 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644724 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644737 2024-11-19T17:50:44,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,752 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644737) roll requested 2024-11-19T17:50:44,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644752 2024-11-19T17:50:44,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,762 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,762 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644737 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644752 2024-11-19T17:50:44,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,768 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644752) roll requested 2024-11-19T17:50:44,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644768 2024-11-19T17:50:44,776 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,776 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,776 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,776 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,777 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,777 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644752 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644768 2024-11-19T17:50:44,777 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,777 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,777 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644768) roll requested 2024-11-19T17:50:44,778 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644777 2024-11-19T17:50:44,791 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,791 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,791 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,791 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,792 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,792 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644768 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644777 2024-11-19T17:50:44,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,799 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644777) roll requested 2024-11-19T17:50:44,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644799 2024-11-19T17:50:44,816 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,816 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,816 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,817 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,817 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,817 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644777 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644799 2024-11-19T17:50:44,817 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,817 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,817 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,817 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644799) roll requested 2024-11-19T17:50:44,818 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644817 2024-11-19T17:50:44,831 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,832 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,832 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,832 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,832 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,832 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644799 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644817 2024-11-19T17:50:44,832 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,832 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,833 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,833 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644817) roll requested 2024-11-19T17:50:44,833 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644833 2024-11-19T17:50:44,842 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,842 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,842 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,842 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,842 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,842 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644817 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644833 2024-11-19T17:50:44,844 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,844 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,844 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,844 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644833) roll requested 2024-11-19T17:50:44,844 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644844 2024-11-19T17:50:44,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,862 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,862 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644833 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644844 2024-11-19T17:50:44,863 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,863 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,863 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,863 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644844) roll requested 2024-11-19T17:50:44,863 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644863 2024-11-19T17:50:44,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,874 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,874 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,874 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,874 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,874 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644844 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644863 2024-11-19T17:50:44,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,876 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644863) roll requested 2024-11-19T17:50:44,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644876 2024-11-19T17:50:44,882 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,883 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,883 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,883 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,883 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,883 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644863 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644876 2024-11-19T17:50:44,889 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,889 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,889 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,889 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644876) roll requested 2024-11-19T17:50:44,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644889 2024-11-19T17:50:44,898 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,898 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644876 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644889 2024-11-19T17:50:44,904 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,904 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,904 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,904 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644889) roll requested 2024-11-19T17:50:44,905 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644905 2024-11-19T17:50:44,910 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,911 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,911 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,911 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,911 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,911 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644889 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644905 2024-11-19T17:50:44,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,912 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,912 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,912 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644905) roll requested 2024-11-19T17:50:44,912 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644912 2024-11-19T17:50:44,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,918 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644905 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644912 2024-11-19T17:50:44,924 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,924 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,924 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644912) roll requested 2024-11-19T17:50:44,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644924 2024-11-19T17:50:44,933 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,933 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,933 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,933 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,933 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,933 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644912 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644924 2024-11-19T17:50:44,936 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,936 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,936 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,936 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644924) roll requested 2024-11-19T17:50:44,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644936 2024-11-19T17:50:44,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,942 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,942 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,942 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,942 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,942 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644924 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644936 2024-11-19T17:50:44,944 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,945 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644936) roll requested 2024-11-19T17:50:44,946 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644945 2024-11-19T17:50:44,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644936 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644945 2024-11-19T17:50:44,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,960 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644945) roll requested 2024-11-19T17:50:44,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644960 2024-11-19T17:50:44,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,966 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,966 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644945 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644960 2024-11-19T17:50:44,971 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,971 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,971 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,971 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644960) roll requested 2024-11-19T17:50:44,971 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644971 2024-11-19T17:50:44,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644960 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644971 2024-11-19T17:50:44,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,980 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644971) roll requested 2024-11-19T17:50:44,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644980 2024-11-19T17:50:44,986 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,986 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,987 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,987 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,987 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,987 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644971 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644980 2024-11-19T17:50:44,988 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:44,988 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,988 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:44,988 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644980) roll requested 2024-11-19T17:50:44,988 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038644988 2024-11-19T17:50:44,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,997 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,997 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,997 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,997 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:44,997 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644980 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644988 2024-11-19T17:50:45,000 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,000 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,001 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038644988) roll requested 2024-11-19T17:50:45,001 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645001 2024-11-19T17:50:45,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,010 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,010 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038644988 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645001 2024-11-19T17:50:45,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,016 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645001) roll requested 2024-11-19T17:50:45,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645017 2024-11-19T17:50:45,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,027 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,027 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,027 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,027 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,027 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645001 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645017 2024-11-19T17:50:45,033 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,033 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,033 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645017) roll requested 2024-11-19T17:50:45,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645033 2024-11-19T17:50:45,041 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,041 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,041 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,041 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,041 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645017 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645033 2024-11-19T17:50:45,042 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,042 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,042 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,042 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645033) roll requested 2024-11-19T17:50:45,042 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645042 2024-11-19T17:50:45,050 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,051 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,051 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,051 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,051 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,051 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645033 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645042 2024-11-19T17:50:45,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,052 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645042) roll requested 2024-11-19T17:50:45,052 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645052 2024-11-19T17:50:45,064 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,064 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,064 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,064 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,065 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,065 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645042 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645052 2024-11-19T17:50:45,069 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,069 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,069 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645052) roll requested 2024-11-19T17:50:45,069 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645069 2024-11-19T17:50:45,075 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,075 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645052 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645069 2024-11-19T17:50:45,076 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,076 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645069) roll requested 2024-11-19T17:50:45,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645077 2024-11-19T17:50:45,085 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,085 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,086 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,086 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,086 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,086 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645069 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645077 2024-11-19T17:50:45,086 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,086 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,086 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,086 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645077) roll requested 2024-11-19T17:50:45,087 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645087 2024-11-19T17:50:45,100 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,100 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,100 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,100 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,101 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,101 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645077 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645087 2024-11-19T17:50:45,102 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,102 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,103 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,103 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645087) roll requested 2024-11-19T17:50:45,103 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645103 2024-11-19T17:50:45,110 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,110 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,110 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,110 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,111 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,111 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645087 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645103 2024-11-19T17:50:45,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,112 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,112 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,112 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645103) roll requested 2024-11-19T17:50:45,112 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645112 2024-11-19T17:50:45,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645103 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645112 2024-11-19T17:50:45,121 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,122 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,122 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,122 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645112) roll requested 2024-11-19T17:50:45,122 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645122 2024-11-19T17:50:45,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,137 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,137 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,137 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,137 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645112 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645122 2024-11-19T17:50:45,138 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,140 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,140 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,140 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645122) roll requested 2024-11-19T17:50:45,140 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645140 2024-11-19T17:50:45,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,148 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,148 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,148 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645122 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645140 2024-11-19T17:50:45,149 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,149 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645140) roll requested 2024-11-19T17:50:45,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645149 2024-11-19T17:50:45,155 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,156 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,156 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,156 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,156 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,157 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645140 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645149 2024-11-19T17:50:45,157 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,157 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,157 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645149) roll requested 2024-11-19T17:50:45,157 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645157 2024-11-19T17:50:45,166 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,166 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,168 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,168 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,168 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645149 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645157 2024-11-19T17:50:45,169 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,176 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,176 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645157) roll requested 2024-11-19T17:50:45,176 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645176 2024-11-19T17:50:45,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,197 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,197 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645157 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645176 2024-11-19T17:50:45,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,197 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,198 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,198 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645176) roll requested 2024-11-19T17:50:45,198 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645198 2024-11-19T17:50:45,203 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,203 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,203 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,203 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,203 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645176 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645198 2024-11-19T17:50:45,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,225 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645198) roll requested 2024-11-19T17:50:45,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645225 2024-11-19T17:50:45,238 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,238 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,238 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,238 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,239 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,239 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645198 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645225 2024-11-19T17:50:45,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,240 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,241 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,241 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645225) roll requested 2024-11-19T17:50:45,241 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645241 2024-11-19T17:50:45,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645225 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645241 2024-11-19T17:50:45,261 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,261 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645241) roll requested 2024-11-19T17:50:45,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645261 2024-11-19T17:50:45,266 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,266 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,266 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,266 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,266 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,266 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645241 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645261 2024-11-19T17:50:45,267 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,267 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,267 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,267 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645261) roll requested 2024-11-19T17:50:45,267 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645267 2024-11-19T17:50:45,273 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,273 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,274 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645261 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645267 2024-11-19T17:50:45,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,285 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645267) roll requested 2024-11-19T17:50:45,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645285 2024-11-19T17:50:45,296 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,296 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,296 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,296 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,297 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,297 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645267 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645285 2024-11-19T17:50:45,305 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,305 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645285) roll requested 2024-11-19T17:50:45,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645305 2024-11-19T17:50:45,316 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,316 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,317 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,317 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645285 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645305 2024-11-19T17:50:45,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,320 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645305) roll requested 2024-11-19T17:50:45,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645320 2024-11-19T17:50:45,332 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,332 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,332 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,332 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,332 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645305 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645320 2024-11-19T17:50:45,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,336 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645320) roll requested 2024-11-19T17:50:45,337 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645337 2024-11-19T17:50:45,343 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645320 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645337 2024-11-19T17:50:45,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,344 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,345 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645337) roll requested 2024-11-19T17:50:45,345 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645345 2024-11-19T17:50:45,351 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,351 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,351 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,352 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,352 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,352 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645337 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645345 2024-11-19T17:50:45,353 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,353 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,354 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,354 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645345) roll requested 2024-11-19T17:50:45,354 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645354 2024-11-19T17:50:45,362 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,362 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645345 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645354 2024-11-19T17:50:45,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,363 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645354) roll requested 2024-11-19T17:50:45,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645363 2024-11-19T17:50:45,381 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,381 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,385 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,385 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,385 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645354 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645363 2024-11-19T17:50:45,386 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,386 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,386 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,386 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645363) roll requested 2024-11-19T17:50:45,386 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645386 2024-11-19T17:50:45,394 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,394 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,394 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,394 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,394 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,394 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645363 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645386 2024-11-19T17:50:45,395 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,395 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,395 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,395 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645386) roll requested 2024-11-19T17:50:45,395 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645395 2024-11-19T17:50:45,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,403 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,403 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,403 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645386 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645395 2024-11-19T17:50:45,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,408 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,409 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,409 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645395) roll requested 2024-11-19T17:50:45,409 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645409 2024-11-19T17:50:45,418 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,418 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,418 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,418 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,418 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,418 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645395 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645409 2024-11-19T17:50:45,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,424 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645409) roll requested 2024-11-19T17:50:45,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645424 2024-11-19T17:50:45,430 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,430 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,430 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,430 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,430 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645409 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645424 2024-11-19T17:50:45,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,432 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,432 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645424) roll requested 2024-11-19T17:50:45,432 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645432 2024-11-19T17:50:45,438 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,438 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,438 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,438 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,438 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645424 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645432 2024-11-19T17:50:45,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,440 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645432) roll requested 2024-11-19T17:50:45,441 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645441 2024-11-19T17:50:45,448 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,448 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,448 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,449 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645432 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645441 2024-11-19T17:50:45,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,452 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,452 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645441) roll requested 2024-11-19T17:50:45,452 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645452 2024-11-19T17:50:45,461 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,461 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,461 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,461 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,461 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,461 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645441 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645452 2024-11-19T17:50:45,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,464 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645452) roll requested 2024-11-19T17:50:45,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645464 2024-11-19T17:50:45,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645452 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645464 2024-11-19T17:50:45,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,472 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645464) roll requested 2024-11-19T17:50:45,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645473 2024-11-19T17:50:45,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,483 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,483 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,484 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,484 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645464 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645473 2024-11-19T17:50:45,488 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,488 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,488 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,488 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645473) roll requested 2024-11-19T17:50:45,489 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645488 2024-11-19T17:50:45,494 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,494 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,494 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,494 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,494 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,494 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645473 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645488 2024-11-19T17:50:45,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,496 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645488) roll requested 2024-11-19T17:50:45,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645496 2024-11-19T17:50:45,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,520 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645488 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645496 2024-11-19T17:50:45,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,524 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,524 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,524 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645496) roll requested 2024-11-19T17:50:45,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645524 2024-11-19T17:50:45,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645496 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645524 2024-11-19T17:50:45,537 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,537 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645524) roll requested 2024-11-19T17:50:45,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645537 2024-11-19T17:50:45,544 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,544 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,544 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,544 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,545 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,545 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645524 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645537 2024-11-19T17:50:45,545 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,545 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,545 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,545 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645537) roll requested 2024-11-19T17:50:45,546 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645546 2024-11-19T17:50:45,554 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,554 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,554 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,554 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,554 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,554 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645537 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645546 2024-11-19T17:50:45,555 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,555 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,555 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,555 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645546) roll requested 2024-11-19T17:50:45,555 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645555 2024-11-19T17:50:45,561 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,561 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,561 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,561 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,561 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,562 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645546 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645555 2024-11-19T17:50:45,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,564 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,564 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645555) roll requested 2024-11-19T17:50:45,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645565 2024-11-19T17:50:45,580 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,580 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,580 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,580 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,581 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645555 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645565 2024-11-19T17:50:45,581 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,581 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,581 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645565) roll requested 2024-11-19T17:50:45,581 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645581 2024-11-19T17:50:45,590 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,591 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,591 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645565 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645581 2024-11-19T17:50:45,591 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,591 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,591 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645581) roll requested 2024-11-19T17:50:45,592 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645592 2024-11-19T17:50:45,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645581 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645592 2024-11-19T17:50:45,603 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,603 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,603 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,603 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645592) roll requested 2024-11-19T17:50:45,603 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645603 2024-11-19T17:50:45,610 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,610 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,611 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,611 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,611 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,612 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645592 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645603 2024-11-19T17:50:45,625 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,625 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645603) roll requested 2024-11-19T17:50:45,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645625 2024-11-19T17:50:45,638 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,638 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,638 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645603 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645625 2024-11-19T17:50:45,639 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,639 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,639 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,639 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645625) roll requested 2024-11-19T17:50:45,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645639 2024-11-19T17:50:45,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,645 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,645 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,645 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645625 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645639 2024-11-19T17:50:45,652 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,652 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,652 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,653 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645639) roll requested 2024-11-19T17:50:45,653 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645653 2024-11-19T17:50:45,658 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,659 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,659 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,659 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,659 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645639 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645653 2024-11-19T17:50:45,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,672 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,673 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645653) roll requested 2024-11-19T17:50:45,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645673 2024-11-19T17:50:45,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645653 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645673 2024-11-19T17:50:45,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,684 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,684 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,685 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645673) roll requested 2024-11-19T17:50:45,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645685 2024-11-19T17:50:45,693 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,693 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,697 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,697 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,697 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645673 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645685 2024-11-19T17:50:45,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,701 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645685) roll requested 2024-11-19T17:50:45,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645701 2024-11-19T17:50:45,712 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,713 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645685 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645701 2024-11-19T17:50:45,714 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,715 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,715 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,715 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645701) roll requested 2024-11-19T17:50:45,715 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645715 2024-11-19T17:50:45,720 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,720 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,721 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645701 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645715 2024-11-19T17:50:45,728 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,728 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,728 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,728 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645715) roll requested 2024-11-19T17:50:45,728 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645728 2024-11-19T17:50:45,753 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,753 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,753 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,753 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,753 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,753 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645715 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645728 2024-11-19T17:50:45,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,756 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645728) roll requested 2024-11-19T17:50:45,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645757 2024-11-19T17:50:45,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645728 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645757 2024-11-19T17:50:45,774 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,774 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,774 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,774 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645757) roll requested 2024-11-19T17:50:45,775 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645775 2024-11-19T17:50:45,786 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,787 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645757 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645775 2024-11-19T17:50:45,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,788 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645775) roll requested 2024-11-19T17:50:45,789 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645789 2024-11-19T17:50:45,796 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,796 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,796 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,796 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,797 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,797 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645775 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645789 2024-11-19T17:50:45,801 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,801 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,801 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,801 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645789) roll requested 2024-11-19T17:50:45,802 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645801 2024-11-19T17:50:45,808 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,808 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,808 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,808 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,809 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,809 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645789 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645801 2024-11-19T17:50:45,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,812 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645801) roll requested 2024-11-19T17:50:45,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645813 2024-11-19T17:50:45,818 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,819 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645801 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645813 2024-11-19T17:50:45,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,821 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645813) roll requested 2024-11-19T17:50:45,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645822 2024-11-19T17:50:45,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,828 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,828 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,828 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,828 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645813 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645822 2024-11-19T17:50:45,829 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,829 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,829 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645822) roll requested 2024-11-19T17:50:45,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645829 2024-11-19T17:50:45,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,836 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,836 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,836 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645822 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645829 2024-11-19T17:50:45,841 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,841 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,841 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,841 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645829) roll requested 2024-11-19T17:50:45,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645841 2024-11-19T17:50:45,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,849 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,849 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,849 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,849 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,850 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645829 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645841 2024-11-19T17:50:45,851 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,851 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,851 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,851 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645841) roll requested 2024-11-19T17:50:45,851 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645851 2024-11-19T17:50:45,858 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,858 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645841 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645851 2024-11-19T17:50:45,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,865 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,865 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645851) roll requested 2024-11-19T17:50:45,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645865 2024-11-19T17:50:45,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,873 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,873 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,873 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645851 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645865 2024-11-19T17:50:45,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,873 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,873 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645865) roll requested 2024-11-19T17:50:45,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645874 2024-11-19T17:50:45,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,881 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,882 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,882 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645865 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645874 2024-11-19T17:50:45,888 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,888 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,888 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645874) roll requested 2024-11-19T17:50:45,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645888 2024-11-19T17:50:45,895 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,895 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,895 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,895 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,895 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,896 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645874 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645888 2024-11-19T17:50:45,897 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,897 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,897 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,897 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645888) roll requested 2024-11-19T17:50:45,897 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645897 2024-11-19T17:50:45,915 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,915 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,915 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,915 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,915 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,915 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645888 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645897 2024-11-19T17:50:45,916 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,916 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,916 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645897) roll requested 2024-11-19T17:50:45,917 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645917 2024-11-19T17:50:45,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,924 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645897 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645917 2024-11-19T17:50:45,933 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,934 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,934 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,934 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645917) roll requested 2024-11-19T17:50:45,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645934 2024-11-19T17:50:45,953 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,953 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,953 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,953 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,953 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:45,953 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645917 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645934 2024-11-19T17:50:45,956 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:45,956 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645917 is not closed yet, will try archiving it next time 2024-11-19T17:50:45,956 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,957 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:45,957 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645934) roll requested 2024-11-19T17:50:45,957 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038645957 2024-11-19T17:50:46,006 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,006 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,006 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,007 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,007 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,007 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645934 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645957 2024-11-19T17:50:46,007 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,007 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038645957) roll requested 2024-11-19T17:50:46,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646007 2024-11-19T17:50:46,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,018 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,018 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038645957 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646007 2024-11-19T17:50:46,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,020 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,020 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,020 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646007) roll requested 2024-11-19T17:50:46,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646021 2024-11-19T17:50:46,032 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,032 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,033 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,033 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,033 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,033 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646007 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646021 2024-11-19T17:50:46,033 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,033 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,034 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,034 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646021) roll requested 2024-11-19T17:50:46,034 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646034 2024-11-19T17:50:46,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,041 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,042 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646021 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646034 2024-11-19T17:50:46,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,044 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646034) roll requested 2024-11-19T17:50:46,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646045 2024-11-19T17:50:46,051 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,051 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,051 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,052 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,052 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,052 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646034 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646045 2024-11-19T17:50:46,054 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,054 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,054 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,054 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646045) roll requested 2024-11-19T17:50:46,055 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646055 2024-11-19T17:50:46,072 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,072 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,073 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,073 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,073 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,073 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646045 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646055 2024-11-19T17:50:46,088 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,088 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,088 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,088 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646055) roll requested 2024-11-19T17:50:46,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646088 2024-11-19T17:50:46,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,106 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646055 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646088 2024-11-19T17:50:46,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,116 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646088) roll requested 2024-11-19T17:50:46,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646117 2024-11-19T17:50:46,131 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,131 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,131 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,131 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,131 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,131 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646088 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646117 2024-11-19T17:50:46,140 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,140 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,140 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,140 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646117) roll requested 2024-11-19T17:50:46,141 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646141 2024-11-19T17:50:46,153 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,153 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,153 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,153 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,153 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,154 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646117 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646141 2024-11-19T17:50:46,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,160 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646141) roll requested 2024-11-19T17:50:46,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646160 2024-11-19T17:50:46,168 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,168 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,168 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,168 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,169 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,169 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646141 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646160 2024-11-19T17:50:46,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,180 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646160) roll requested 2024-11-19T17:50:46,181 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646180 2024-11-19T17:50:46,200 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,200 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,200 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,200 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,200 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646160 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646180 2024-11-19T17:50:46,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,204 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,205 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646180) roll requested 2024-11-19T17:50:46,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646205 2024-11-19T17:50:46,216 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,216 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,216 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,217 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,217 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,217 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646180 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646205 2024-11-19T17:50:46,220 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,220 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,220 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,220 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646205) roll requested 2024-11-19T17:50:46,221 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646220 2024-11-19T17:50:46,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,236 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,236 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,236 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646205 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646220 2024-11-19T17:50:46,237 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,237 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,237 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646220) roll requested 2024-11-19T17:50:46,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646237 2024-11-19T17:50:46,246 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,246 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,247 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646220 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646237 2024-11-19T17:50:46,247 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,247 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,247 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,247 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646237) roll requested 2024-11-19T17:50:46,247 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646247 2024-11-19T17:50:46,254 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,254 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,255 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646237 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646247 2024-11-19T17:50:46,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,256 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,256 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,256 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646247) roll requested 2024-11-19T17:50:46,256 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646256 2024-11-19T17:50:46,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,263 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,263 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646247 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646256 2024-11-19T17:50:46,263 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,263 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,263 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,263 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646256) roll requested 2024-11-19T17:50:46,264 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646264 2024-11-19T17:50:46,273 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,273 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,273 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,273 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,273 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,273 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646256 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646264 2024-11-19T17:50:46,274 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,274 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,274 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,274 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646264) roll requested 2024-11-19T17:50:46,274 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646274 2024-11-19T17:50:46,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,290 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,290 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,290 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,290 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,290 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646264 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646274 2024-11-19T17:50:46,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,300 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646274) roll requested 2024-11-19T17:50:46,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646301 2024-11-19T17:50:46,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,320 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646274 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646301 2024-11-19T17:50:46,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,320 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646301) roll requested 2024-11-19T17:50:46,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646320 2024-11-19T17:50:46,328 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,329 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,329 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,329 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646301 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646320 2024-11-19T17:50:46,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,332 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,333 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646320) roll requested 2024-11-19T17:50:46,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646333 2024-11-19T17:50:46,345 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,352 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,352 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,352 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,352 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646320 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646333 2024-11-19T17:50:46,376 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,376 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,376 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,377 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646333) roll requested 2024-11-19T17:50:46,377 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646377 2024-11-19T17:50:46,386 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,386 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,386 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,386 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,387 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646333 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646377 2024-11-19T17:50:46,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,396 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,396 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646377) roll requested 2024-11-19T17:50:46,396 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646396 2024-11-19T17:50:46,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,403 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,403 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,403 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646377 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646396 2024-11-19T17:50:46,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,415 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646396) roll requested 2024-11-19T17:50:46,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646415 2024-11-19T17:50:46,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,421 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,421 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646396 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646415 2024-11-19T17:50:46,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,424 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646415) roll requested 2024-11-19T17:50:46,425 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646425 2024-11-19T17:50:46,442 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,442 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,442 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,442 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,442 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,442 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646415 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646425 2024-11-19T17:50:46,444 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,446 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,446 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,446 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646425) roll requested 2024-11-19T17:50:46,446 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646446 2024-11-19T17:50:46,453 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,453 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,453 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,455 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,455 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646425 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646446 2024-11-19T17:50:46,463 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,463 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,463 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,463 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646446) roll requested 2024-11-19T17:50:46,463 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646463 2024-11-19T17:50:46,475 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,475 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,476 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646446 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646463 2024-11-19T17:50:46,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,480 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,481 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646463) roll requested 2024-11-19T17:50:46,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646481 2024-11-19T17:50:46,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,490 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,491 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,491 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,491 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646463 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646481 2024-11-19T17:50:46,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,492 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,492 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,492 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646481) roll requested 2024-11-19T17:50:46,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646493 2024-11-19T17:50:46,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,501 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646481 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646493 2024-11-19T17:50:46,505 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,505 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,505 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646493) roll requested 2024-11-19T17:50:46,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646505 2024-11-19T17:50:46,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646493 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646505 2024-11-19T17:50:46,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,512 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,512 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646505) roll requested 2024-11-19T17:50:46,512 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646512 2024-11-19T17:50:46,522 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,522 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,523 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,523 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646505 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646512 2024-11-19T17:50:46,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,524 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,524 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,524 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646512) roll requested 2024-11-19T17:50:46,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646525 2024-11-19T17:50:46,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,535 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646512 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646525 2024-11-19T17:50:46,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,536 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,536 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,536 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646525) roll requested 2024-11-19T17:50:46,536 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646536 2024-11-19T17:50:46,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,549 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646525 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646536 2024-11-19T17:50:46,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,552 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646536) roll requested 2024-11-19T17:50:46,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646552 2024-11-19T17:50:46,567 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,567 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,567 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,567 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,567 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,567 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646536 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646552 2024-11-19T17:50:46,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,568 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,569 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,569 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646552) roll requested 2024-11-19T17:50:46,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646569 2024-11-19T17:50:46,579 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,580 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,580 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,580 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,580 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,580 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646552 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646569 2024-11-19T17:50:46,581 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,581 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,581 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646569) roll requested 2024-11-19T17:50:46,582 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646582 2024-11-19T17:50:46,587 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,587 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,587 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646569 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646582 2024-11-19T17:50:46,588 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,588 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,588 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,588 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646582) roll requested 2024-11-19T17:50:46,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646588 2024-11-19T17:50:46,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646582 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646588 2024-11-19T17:50:46,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,604 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646588) roll requested 2024-11-19T17:50:46,605 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646604 2024-11-19T17:50:46,609 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,610 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,610 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,610 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,610 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646588 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646604 2024-11-19T17:50:46,610 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,610 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,610 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,610 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646604) roll requested 2024-11-19T17:50:46,611 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646610 2024-11-19T17:50:46,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,616 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,616 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,616 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646604 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646610 2024-11-19T17:50:46,616 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,616 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,617 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646610) roll requested 2024-11-19T17:50:46,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646617 2024-11-19T17:50:46,621 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,622 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,622 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,622 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,622 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646610 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646617 2024-11-19T17:50:46,631 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,631 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,631 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,631 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646617) roll requested 2024-11-19T17:50:46,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646631 2024-11-19T17:50:46,636 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,636 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,637 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,637 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,637 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,637 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646617 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646631 2024-11-19T17:50:46,642 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,642 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,642 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,642 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646631) roll requested 2024-11-19T17:50:46,642 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646642 2024-11-19T17:50:46,653 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,654 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,654 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,654 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646631 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646642 2024-11-19T17:50:46,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,664 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646642) roll requested 2024-11-19T17:50:46,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646664 2024-11-19T17:50:46,670 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,670 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,670 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,670 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,670 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,670 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646642 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646664 2024-11-19T17:50:46,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,676 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,677 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,677 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646664) roll requested 2024-11-19T17:50:46,677 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646677 2024-11-19T17:50:46,683 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,683 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,683 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,683 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,683 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,683 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646664 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646677 2024-11-19T17:50:46,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,684 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,684 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,684 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646677) roll requested 2024-11-19T17:50:46,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646685 2024-11-19T17:50:46,691 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,691 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,691 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,691 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,691 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,691 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646677 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646685 2024-11-19T17:50:46,692 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,692 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,693 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,693 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646685) roll requested 2024-11-19T17:50:46,693 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646693 2024-11-19T17:50:46,699 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,699 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,699 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,699 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,700 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646685 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646693 2024-11-19T17:50:46,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,701 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,702 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,702 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646693) roll requested 2024-11-19T17:50:46,702 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646702 2024-11-19T17:50:46,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,711 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,711 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,711 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,711 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646693 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646702 2024-11-19T17:50:46,712 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,712 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,712 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,712 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646702) roll requested 2024-11-19T17:50:46,712 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646712 2024-11-19T17:50:46,718 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,718 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,718 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,718 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,718 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,718 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646702 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646712 2024-11-19T17:50:46,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,728 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,728 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,728 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646712) roll requested 2024-11-19T17:50:46,728 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646728 2024-11-19T17:50:46,736 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,736 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,736 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,736 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,737 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,737 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646712 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646728 2024-11-19T17:50:46,737 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,737 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,737 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646728) roll requested 2024-11-19T17:50:46,737 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646737 2024-11-19T17:50:46,742 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,742 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,742 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,743 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,743 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,743 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646728 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646737 2024-11-19T17:50:46,743 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,743 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646737) roll requested 2024-11-19T17:50:46,744 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646744 2024-11-19T17:50:46,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,750 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,750 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646737 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646744 2024-11-19T17:50:46,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,756 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646744) roll requested 2024-11-19T17:50:46,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646757 2024-11-19T17:50:46,764 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,764 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,764 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,765 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,765 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,765 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646744 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646757 2024-11-19T17:50:46,765 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,765 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,765 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,766 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646757) roll requested 2024-11-19T17:50:46,766 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646766 2024-11-19T17:50:46,774 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,776 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,777 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,777 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646757 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646766 2024-11-19T17:50:46,780 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,780 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,781 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,781 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646766) roll requested 2024-11-19T17:50:46,781 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646781 2024-11-19T17:50:46,788 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,788 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,789 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,789 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,789 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,789 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646766 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646781 2024-11-19T17:50:46,790 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,790 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,790 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,790 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646781) roll requested 2024-11-19T17:50:46,790 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646790 2024-11-19T17:50:46,808 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,808 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,809 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,809 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,809 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,809 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646781 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646790 2024-11-19T17:50:46,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,812 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646790) roll requested 2024-11-19T17:50:46,812 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646812 2024-11-19T17:50:46,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646790 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646812 2024-11-19T17:50:46,839 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,839 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,839 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,839 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646812) roll requested 2024-11-19T17:50:46,839 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646839 2024-11-19T17:50:46,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646812 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646839 2024-11-19T17:50:46,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,848 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,848 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646839) roll requested 2024-11-19T17:50:46,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646848 2024-11-19T17:50:46,860 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,860 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,861 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,861 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,861 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,861 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646839 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646848 2024-11-19T17:50:46,861 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,861 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,861 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,861 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646848) roll requested 2024-11-19T17:50:46,862 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646861 2024-11-19T17:50:46,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,872 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,872 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646848 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646861 2024-11-19T17:50:46,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,876 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646861) roll requested 2024-11-19T17:50:46,877 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646876 2024-11-19T17:50:46,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,887 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,887 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,887 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646861 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646876 2024-11-19T17:50:46,888 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,888 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,888 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646876) roll requested 2024-11-19T17:50:46,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646888 2024-11-19T17:50:46,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646876 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646888 2024-11-19T17:50:46,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,901 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646888) roll requested 2024-11-19T17:50:46,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646901 2024-11-19T17:50:46,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,913 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,914 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,914 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646888 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646901 2024-11-19T17:50:46,916 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,916 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,916 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,916 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646901) roll requested 2024-11-19T17:50:46,917 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646916 2024-11-19T17:50:46,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,924 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646901 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646916 2024-11-19T17:50:46,931 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,931 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,931 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646916) roll requested 2024-11-19T17:50:46,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646931 2024-11-19T17:50:46,938 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,938 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,938 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,938 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,938 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,939 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646916 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646931 2024-11-19T17:50:46,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,940 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646931) roll requested 2024-11-19T17:50:46,941 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646941 2024-11-19T17:50:46,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646931 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646941 2024-11-19T17:50:46,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,968 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,968 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646941) roll requested 2024-11-19T17:50:46,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646968 2024-11-19T17:50:46,975 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,975 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,975 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,975 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,975 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,975 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646941 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646968 2024-11-19T17:50:46,976 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,976 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,976 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,976 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646968) roll requested 2024-11-19T17:50:46,976 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646976 2024-11-19T17:50:46,985 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,985 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,986 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,986 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,986 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,986 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646968 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646976 2024-11-19T17:50:46,986 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,986 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,986 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,986 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646976) roll requested 2024-11-19T17:50:46,987 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646986 2024-11-19T17:50:46,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,995 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:46,995 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646976 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646986 2024-11-19T17:50:46,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:46,996 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,996 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:46,996 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646986) roll requested 2024-11-19T17:50:46,996 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038646996 2024-11-19T17:50:47,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,004 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,004 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,004 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646986 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646996 2024-11-19T17:50:47,008 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,008 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,008 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,008 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038646996) roll requested 2024-11-19T17:50:47,009 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647008 2024-11-19T17:50:47,015 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,015 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,015 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,015 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,015 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038646996 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647008 2024-11-19T17:50:47,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,016 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,016 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647008) roll requested 2024-11-19T17:50:47,016 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647016 2024-11-19T17:50:47,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647008 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647016 2024-11-19T17:50:47,024 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,024 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,024 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,024 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647016) roll requested 2024-11-19T17:50:47,024 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647024 2024-11-19T17:50:47,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647016 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647024 2024-11-19T17:50:47,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,032 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647024) roll requested 2024-11-19T17:50:47,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647032 2024-11-19T17:50:47,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,041 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,041 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647024 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647032 2024-11-19T17:50:47,042 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,042 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,042 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,042 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647032) roll requested 2024-11-19T17:50:47,042 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647042 2024-11-19T17:50:47,051 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,051 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,051 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,052 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,052 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,052 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647032 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647042 2024-11-19T17:50:47,054 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,054 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,054 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,054 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647042) roll requested 2024-11-19T17:50:47,054 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647054 2024-11-19T17:50:47,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,060 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,060 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,060 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647042 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647054 2024-11-19T17:50:47,061 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,061 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647042 is not closed yet, will try archiving it next time 2024-11-19T17:50:47,061 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,061 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,061 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647054) roll requested 2024-11-19T17:50:47,061 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647061 2024-11-19T17:50:47,066 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,067 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647054 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647061 2024-11-19T17:50:47,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647054 is not closed yet, will try archiving it next time 2024-11-19T17:50:47,068 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,068 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,068 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647061) roll requested 2024-11-19T17:50:47,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647068 2024-11-19T17:50:47,075 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,075 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647061 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647068 2024-11-19T17:50:47,086 DEBUG [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647061 is not closed yet, will try archiving it next time 2024-11-19T17:50:47,088 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,088 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,088 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,088 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647068) roll requested 2024-11-19T17:50:47,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647088 2024-11-19T17:50:47,101 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,101 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,101 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,101 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,101 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,101 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647068 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647088 2024-11-19T17:50:47,104 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,104 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647088) roll requested 2024-11-19T17:50:47,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647105 2024-11-19T17:50:47,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,113 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647088 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647105 2024-11-19T17:50:47,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,116 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647105) roll requested 2024-11-19T17:50:47,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647116 2024-11-19T17:50:47,123 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,123 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647105 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647116 2024-11-19T17:50:47,124 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,124 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,124 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,124 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647116) roll requested 2024-11-19T17:50:47,125 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647125 2024-11-19T17:50:47,130 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,131 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,131 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,131 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,131 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,131 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647116 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647125 2024-11-19T17:50:47,132 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,132 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,132 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,132 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647125) roll requested 2024-11-19T17:50:47,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647133 2024-11-19T17:50:47,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,142 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,142 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647125 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647133 2024-11-19T17:50:47,145 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,145 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647133) roll requested 2024-11-19T17:50:47,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647145 2024-11-19T17:50:47,153 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,153 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,153 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,153 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,154 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,154 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647133 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647145 2024-11-19T17:50:47,156 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,156 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,156 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647145) roll requested 2024-11-19T17:50:47,157 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647157 2024-11-19T17:50:47,162 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,162 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,162 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,162 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,162 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,162 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647145 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647157 2024-11-19T17:50:47,162 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,162 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,163 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,163 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647157) roll requested 2024-11-19T17:50:47,163 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647163 2024-11-19T17:50:47,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,172 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,172 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647157 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647163 2024-11-19T17:50:47,173 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,173 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,173 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647163) roll requested 2024-11-19T17:50:47,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647173 2024-11-19T17:50:47,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,188 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647163 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647173 2024-11-19T17:50:47,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,188 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647173) roll requested 2024-11-19T17:50:47,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647189 2024-11-19T17:50:47,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,199 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,199 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647173 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647189 2024-11-19T17:50:47,200 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,200 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,200 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,200 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647189) roll requested 2024-11-19T17:50:47,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647200 2024-11-19T17:50:47,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,211 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,211 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647189 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647200 2024-11-19T17:50:47,228 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,230 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,230 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,230 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647200) roll requested 2024-11-19T17:50:47,230 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647230 2024-11-19T17:50:47,244 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,244 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647200 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647230 2024-11-19T17:50:47,248 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,248 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,248 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,248 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647230) roll requested 2024-11-19T17:50:47,248 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647248 2024-11-19T17:50:47,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,259 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,259 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647230 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647248 2024-11-19T17:50:47,261 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,261 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647248) roll requested 2024-11-19T17:50:47,262 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647262 2024-11-19T17:50:47,272 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,272 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,272 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,273 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,273 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,273 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647248 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647262 2024-11-19T17:50:47,294 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,294 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,294 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647262) roll requested 2024-11-19T17:50:47,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647294 2024-11-19T17:50:47,322 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,322 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,322 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,322 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,322 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,322 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647262 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647294 2024-11-19T17:50:47,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,325 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,325 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,325 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647294) roll requested 2024-11-19T17:50:47,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647325 2024-11-19T17:50:47,332 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647294 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647325 2024-11-19T17:50:47,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,334 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647325) roll requested 2024-11-19T17:50:47,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647334 2024-11-19T17:50:47,342 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,344 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,344 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647325 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647334 2024-11-19T17:50:47,352 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,352 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,353 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,353 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647334) roll requested 2024-11-19T17:50:47,353 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647353 2024-11-19T17:50:47,364 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,364 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647334 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647353 2024-11-19T17:50:47,373 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,376 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,376 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,376 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647353) roll requested 2024-11-19T17:50:47,376 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647376 2024-11-19T17:50:47,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647353 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647376 2024-11-19T17:50:47,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,393 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,393 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,393 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647376) roll requested 2024-11-19T17:50:47,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647393 2024-11-19T17:50:47,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,406 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647376 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647393 2024-11-19T17:50:47,407 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,407 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,407 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647393) roll requested 2024-11-19T17:50:47,407 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647407 2024-11-19T17:50:47,428 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,429 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,429 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,429 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647393 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647407 2024-11-19T17:50:47,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,432 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,432 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647407) roll requested 2024-11-19T17:50:47,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647432 2024-11-19T17:50:47,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647407 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647432 2024-11-19T17:50:47,446 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,446 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,446 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,446 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647432) roll requested 2024-11-19T17:50:47,446 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647446 2024-11-19T17:50:47,454 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,454 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,455 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647432 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647446 2024-11-19T17:50:47,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,457 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647446) roll requested 2024-11-19T17:50:47,458 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647457 2024-11-19T17:50:47,467 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,467 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,467 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,467 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,467 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,467 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647446 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647457 2024-11-19T17:50:47,469 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,469 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,469 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,469 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647457) roll requested 2024-11-19T17:50:47,469 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647469 2024-11-19T17:50:47,475 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,476 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,476 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,476 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,476 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,476 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647457 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647469 2024-11-19T17:50:47,485 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,485 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,485 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647469) roll requested 2024-11-19T17:50:47,486 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647486 2024-11-19T17:50:47,491 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,491 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,491 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,491 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,491 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647469 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647486 2024-11-19T17:50:47,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,496 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647486) roll requested 2024-11-19T17:50:47,496 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647496 2024-11-19T17:50:47,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,506 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647486 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647496 2024-11-19T17:50:47,507 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,507 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,507 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,507 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647496) roll requested 2024-11-19T17:50:47,507 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647507 2024-11-19T17:50:47,519 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,519 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,519 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,519 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,519 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,519 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647496 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647507 2024-11-19T17:50:47,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,520 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,520 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647507) roll requested 2024-11-19T17:50:47,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647520 2024-11-19T17:50:47,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647507 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647520 2024-11-19T17:50:47,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,531 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647520) roll requested 2024-11-19T17:50:47,532 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647532 2024-11-19T17:50:47,537 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,537 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,537 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,537 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,537 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,537 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647520 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647532 2024-11-19T17:50:47,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,540 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,540 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,540 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647532) roll requested 2024-11-19T17:50:47,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647540 2024-11-19T17:50:47,550 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,550 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,551 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,551 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,551 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647532 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647540 2024-11-19T17:50:47,551 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,551 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647540) roll requested 2024-11-19T17:50:47,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647552 2024-11-19T17:50:47,557 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,557 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,557 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,557 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,557 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,557 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647540 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647552 2024-11-19T17:50:47,561 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,561 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,561 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,561 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647552) roll requested 2024-11-19T17:50:47,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647561 2024-11-19T17:50:47,566 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,566 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,566 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,567 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,567 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,567 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647552 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647561 2024-11-19T17:50:47,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,568 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,568 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647561) roll requested 2024-11-19T17:50:47,568 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647568 2024-11-19T17:50:47,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,574 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,574 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,574 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,574 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,574 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647561 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647568 2024-11-19T17:50:47,575 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,575 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,575 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,575 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647568) roll requested 2024-11-19T17:50:47,575 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647575 2024-11-19T17:50:47,588 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,589 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,589 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,589 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,589 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647568 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647575 2024-11-19T17:50:47,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,592 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647575) roll requested 2024-11-19T17:50:47,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647593 2024-11-19T17:50:47,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,601 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647575 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647593 2024-11-19T17:50:47,602 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,602 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,602 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,602 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647593) roll requested 2024-11-19T17:50:47,602 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647602 2024-11-19T17:50:47,615 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,615 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,615 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,615 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647593 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647602 2024-11-19T17:50:47,616 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,616 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,616 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,616 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647602) roll requested 2024-11-19T17:50:47,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647616 2024-11-19T17:50:47,624 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,624 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,624 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,625 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,625 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,625 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647602 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647616 2024-11-19T17:50:47,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,628 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647616) roll requested 2024-11-19T17:50:47,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647629 2024-11-19T17:50:47,640 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,640 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,640 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,640 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,641 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,641 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647616 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647629 2024-11-19T17:50:47,642 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,642 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,642 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,642 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647629) roll requested 2024-11-19T17:50:47,642 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647642 2024-11-19T17:50:47,648 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,648 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,649 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,649 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,649 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,649 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647629 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647642 2024-11-19T17:50:47,650 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,650 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,651 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647642) roll requested 2024-11-19T17:50:47,651 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647651 2024-11-19T17:50:47,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,667 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,667 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,667 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647642 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647651 2024-11-19T17:50:47,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,669 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,669 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647651) roll requested 2024-11-19T17:50:47,669 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647669 2024-11-19T17:50:47,698 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,698 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,700 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,700 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,700 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,700 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647651 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647669 2024-11-19T17:50:47,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,717 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,717 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,717 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647669) roll requested 2024-11-19T17:50:47,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647717 2024-11-19T17:50:47,739 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,739 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,739 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647669 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647717 2024-11-19T17:50:47,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,744 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647717) roll requested 2024-11-19T17:50:47,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647744 2024-11-19T17:50:47,757 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,757 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,757 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,757 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,757 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647717 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647744 2024-11-19T17:50:47,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,768 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647744) roll requested 2024-11-19T17:50:47,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647768 2024-11-19T17:50:47,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,785 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,785 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,785 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647744 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647768 2024-11-19T17:50:47,786 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,786 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,786 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,786 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647768) roll requested 2024-11-19T17:50:47,786 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647786 2024-11-19T17:50:47,796 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,796 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,796 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,796 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,796 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,797 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647768 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647786 2024-11-19T17:50:47,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,799 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647786) roll requested 2024-11-19T17:50:47,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647799 2024-11-19T17:50:47,807 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,807 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,808 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,809 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647786 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647799 2024-11-19T17:50:47,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,812 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647799) roll requested 2024-11-19T17:50:47,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647813 2024-11-19T17:50:47,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,830 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,830 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647799 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647813 2024-11-19T17:50:47,830 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,830 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,830 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,830 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647813) roll requested 2024-11-19T17:50:47,831 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647830 2024-11-19T17:50:47,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,849 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,849 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,849 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647813 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647830 2024-11-19T17:50:47,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,856 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647830) roll requested 2024-11-19T17:50:47,857 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647857 2024-11-19T17:50:47,862 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,863 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,863 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647830 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647857 2024-11-19T17:50:47,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,864 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,864 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647857) roll requested 2024-11-19T17:50:47,864 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647864 2024-11-19T17:50:47,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,878 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,878 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,878 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,878 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647857 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647864 2024-11-19T17:50:47,879 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,878 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,879 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,879 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647864) roll requested 2024-11-19T17:50:47,879 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647879 2024-11-19T17:50:47,887 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,889 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,895 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,895 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,896 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,896 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647864 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647879 2024-11-19T17:50:47,896 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,896 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,896 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,896 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647879) roll requested 2024-11-19T17:50:47,896 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647896 2024-11-19T17:50:47,902 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,902 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,902 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,902 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,902 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,902 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647879 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647896 2024-11-19T17:50:47,904 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,904 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,904 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,904 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647896) roll requested 2024-11-19T17:50:47,905 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647904 2024-11-19T17:50:47,910 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,910 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,910 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,910 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,910 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647896 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647904 2024-11-19T17:50:47,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,920 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647904) roll requested 2024-11-19T17:50:47,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647921 2024-11-19T17:50:47,929 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,929 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,929 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,930 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,930 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647904 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647921 2024-11-19T17:50:47,930 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,930 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,930 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647921) roll requested 2024-11-19T17:50:47,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647931 2024-11-19T17:50:47,937 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,937 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,937 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,937 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,937 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,938 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647921 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647931 2024-11-19T17:50:47,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,940 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647931) roll requested 2024-11-19T17:50:47,941 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647941 2024-11-19T17:50:47,962 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,962 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,962 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,962 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,962 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,962 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647931 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647941 2024-11-19T17:50:47,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,968 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,968 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647941) roll requested 2024-11-19T17:50:47,969 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647968 2024-11-19T17:50:47,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647941 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647968 2024-11-19T17:50:47,983 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:47,983 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,984 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:47,984 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647968) roll requested 2024-11-19T17:50:47,984 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038647984 2024-11-19T17:50:47,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,995 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:47,995 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647968 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647984 2024-11-19T17:50:48,004 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:48,004 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038647984) roll requested 2024-11-19T17:50:48,005 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038648004 2024-11-19T17:50:48,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,010 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,010 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038647984 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648004 2024-11-19T17:50:48,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:48,017 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038648004) roll requested 2024-11-19T17:50:48,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038648017 2024-11-19T17:50:48,023 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,023 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648004 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648017 2024-11-19T17:50:48,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:48,025 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038648017) roll requested 2024-11-19T17:50:48,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038648025 2024-11-19T17:50:48,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648017 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648025 2024-11-19T17:50:48,033 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:48,033 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,033 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038648025) roll requested 2024-11-19T17:50:48,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038648033 2024-11-19T17:50:48,039 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,039 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,039 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,040 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648025 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648033 2024-11-19T17:50:48,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:48,040 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,040 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038648033) roll requested 2024-11-19T17:50:48,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038648041 2024-11-19T17:50:48,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,048 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,048 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,048 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648033 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648041 2024-11-19T17:50:48,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:48,048 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,049 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038648041) roll requested 2024-11-19T17:50:48,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038648049 2024-11-19T17:50:48,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,057 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,058 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648041 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648049 2024-11-19T17:50:48,061 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:48,061 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,061 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,061 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038648049) roll requested 2024-11-19T17:50:48,061 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038648061 2024-11-19T17:50:48,081 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,081 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,081 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,082 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,082 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,082 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648049 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648061 2024-11-19T17:50:48,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:48,083 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,083 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038648061) roll requested 2024-11-19T17:50:48,083 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038648083 2024-11-19T17:50:48,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,092 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,094 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648061 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648083 2024-11-19T17:50:48,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:48,116 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038648083) roll requested 2024-11-19T17:50:48,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038648117 2024-11-19T17:50:48,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,122 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,122 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,122 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,122 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,122 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648083 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648117 2024-11-19T17:50:48,124 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:48,124 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,124 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,125 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038648117) roll requested 2024-11-19T17:50:48,125 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038648125 2024-11-19T17:50:48,139 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,140 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,140 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,140 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,140 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,140 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648117 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648125 2024-11-19T17:50:48,144 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:48,144 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,144 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,144 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038648125) roll requested 2024-11-19T17:50:48,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038648144 2024-11-19T17:50:48,155 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,155 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648125 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648144 2024-11-19T17:50:48,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:48,161 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038648144) roll requested 2024-11-19T17:50:48,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038648161 2024-11-19T17:50:48,173 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,173 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,173 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,173 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,173 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,173 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648144 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648161 2024-11-19T17:50:48,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:48,176 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,176 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038648161) roll requested 2024-11-19T17:50:48,176 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038648176 2024-11-19T17:50:48,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,188 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,189 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648161 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648176 2024-11-19T17:50:48,192 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:48,192 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,192 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,193 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038648176) roll requested 2024-11-19T17:50:48,193 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038648193 2024-11-19T17:50:48,200 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,200 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,200 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,201 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648176 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648193 2024-11-19T17:50:48,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:48,204 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,204 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038648193) roll requested 2024-11-19T17:50:48,204 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038648204 2024-11-19T17:50:48,212 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,212 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,212 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648193 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648204 2024-11-19T17:50:48,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:48,216 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,217 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038648204) roll requested 2024-11-19T17:50:48,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038648217 2024-11-19T17:50:48,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,236 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,236 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648204 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648217 2024-11-19T17:50:48,237 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:48,237 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,237 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038648217) roll requested 2024-11-19T17:50:48,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038648237 2024-11-19T17:50:48,247 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,247 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,247 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,247 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,247 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,247 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648217 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648237 2024-11-19T17:50:48,248 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:48,248 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,248 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,248 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038648237) roll requested 2024-11-19T17:50:48,248 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038648248 2024-11-19T17:50:48,268 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,268 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,271 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,271 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648237 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648248 2024-11-19T17:50:48,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:48,272 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,272 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038648248) roll requested 2024-11-19T17:50:48,272 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038648272 2024-11-19T17:50:48,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,283 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648248 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648272 2024-11-19T17:50:48,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:48,284 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,284 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038648272) roll requested 2024-11-19T17:50:48,284 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038648284 2024-11-19T17:50:48,295 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,295 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,295 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,295 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648272 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648284 2024-11-19T17:50:48,303 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:48,303 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038648284) roll requested 2024-11-19T17:50:48,304 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038648304 2024-11-19T17:50:48,316 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,316 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648284 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648304 2024-11-19T17:50:48,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:48,320 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038648304) roll requested 2024-11-19T17:50:48,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038648320 2024-11-19T17:50:48,336 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,336 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,336 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,336 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,337 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,337 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648304 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648320 2024-11-19T17:50:48,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:48,340 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,341 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038648320) roll requested 2024-11-19T17:50:48,341 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038648341 2024-11-19T17:50:48,364 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,364 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,364 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,365 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648320 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648341 2024-11-19T17:50:48,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:48,368 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038648341) roll requested 2024-11-19T17:50:48,369 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038648369 2024-11-19T17:50:48,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,394 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648341 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648369 2024-11-19T17:50:48,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:48,400 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,401 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038648369) roll requested 2024-11-19T17:50:48,401 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038648401 2024-11-19T17:50:48,409 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,412 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648369 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648401 2024-11-19T17:50:48,414 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:48,414 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,414 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,414 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038648401) roll requested 2024-11-19T17:50:48,414 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038648414 2024-11-19T17:50:48,422 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,422 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648401 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648414 2024-11-19T17:50:48,423 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:48,423 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,423 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,423 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038648414) roll requested 2024-11-19T17:50:48,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038648424 2024-11-19T17:50:48,434 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,434 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,434 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,434 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,434 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,435 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648414 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648424 2024-11-19T17:50:48,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:48,441 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038648424) roll requested 2024-11-19T17:50:48,441 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038648441 2024-11-19T17:50:48,468 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,468 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,468 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,469 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,469 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,469 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648424 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648441 2024-11-19T17:50:48,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:48,504 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038648441) roll requested 2024-11-19T17:50:48,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038648504 2024-11-19T17:50:48,528 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,528 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,528 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648441 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648504 2024-11-19T17:50:48,533 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:48,533 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,533 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038648504) roll requested 2024-11-19T17:50:48,534 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038648533 2024-11-19T17:50:48,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,546 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,546 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,546 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648504 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648533 2024-11-19T17:50:48,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:48,549 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038648533) roll requested 2024-11-19T17:50:48,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038648549 2024-11-19T17:50:48,579 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,579 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,580 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,580 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,580 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,580 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648533 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648549 2024-11-19T17:50:48,588 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:48,589 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038648549) roll requested 2024-11-19T17:50:48,590 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038648589 2024-11-19T17:50:48,624 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,624 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,624 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,624 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,625 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648549 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648589 2024-11-19T17:50:48,648 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:48,648 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,648 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038648589) roll requested 2024-11-19T17:50:48,649 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038648649 2024-11-19T17:50:48,654 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,654 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,655 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,655 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,656 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648589 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648649 2024-11-19T17:50:48,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:48,656 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038648649) roll requested 2024-11-19T17:50:48,657 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038648656 2024-11-19T17:50:48,662 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,662 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,662 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648649 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648656 2024-11-19T17:50:48,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:48,664 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038648656) roll requested 2024-11-19T17:50:48,665 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038648664 2024-11-19T17:50:48,670 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,671 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,671 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,671 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,671 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648656 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648664 2024-11-19T17:50:48,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:48,672 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,672 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038648664) roll requested 2024-11-19T17:50:48,672 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038648672 2024-11-19T17:50:48,679 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,679 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,679 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,680 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648664 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648672 2024-11-19T17:50:48,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:48,680 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,681 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038648672) roll requested 2024-11-19T17:50:48,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038648681 2024-11-19T17:50:48,687 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,687 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,687 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,687 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,687 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,687 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648672 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648681 2024-11-19T17:50:48,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:48,688 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,688 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,688 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038648681) roll requested 2024-11-19T17:50:48,688 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038648688 2024-11-19T17:50:48,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648681 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648688 2024-11-19T17:50:48,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:48,704 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038648688) roll requested 2024-11-19T17:50:48,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038648704 2024-11-19T17:50:48,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648688 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648704 2024-11-19T17:50:48,712 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:48,712 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,712 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,713 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038648704) roll requested 2024-11-19T17:50:48,713 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038648713 2024-11-19T17:50:48,718 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,718 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,718 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,718 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,718 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,719 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648704 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648713 2024-11-19T17:50:48,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:48,724 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,724 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038648713) roll requested 2024-11-19T17:50:48,724 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038648724 2024-11-19T17:50:48,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,732 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,732 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648713 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648724 2024-11-19T17:50:48,734 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:48,734 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,734 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,734 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038648724) roll requested 2024-11-19T17:50:48,734 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038648734 2024-11-19T17:50:48,742 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,742 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,743 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,743 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,744 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,744 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648724 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648734 2024-11-19T17:50:48,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:48,744 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038648734) roll requested 2024-11-19T17:50:48,744 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038648744 2024-11-19T17:50:48,749 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,749 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,749 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,749 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,749 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,749 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648734 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648744 2024-11-19T17:50:48,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:48,752 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038648744) roll requested 2024-11-19T17:50:48,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038648752 2024-11-19T17:50:48,774 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,775 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648744 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648752 2024-11-19T17:50:48,786 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,786 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,786 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:48,786 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038648752) roll requested 2024-11-19T17:50:48,786 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038648786 2024-11-19T17:50:48,797 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,797 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,797 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,797 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,797 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,797 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648752 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648786 2024-11-19T17:50:48,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:48,800 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,800 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,800 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038648786) roll requested 2024-11-19T17:50:48,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038648800 2024-11-19T17:50:48,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,821 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648786 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648800 2024-11-19T17:50:48,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:48,828 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038648800) roll requested 2024-11-19T17:50:48,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038648829 2024-11-19T17:50:48,845 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,845 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,845 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,845 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,845 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,845 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648800 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648829 2024-11-19T17:50:48,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:48,869 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,869 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,869 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038648829) roll requested 2024-11-19T17:50:48,869 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038648869 2024-11-19T17:50:48,888 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,888 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,888 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648829 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648869 2024-11-19T17:50:48,904 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:48,904 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,904 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,904 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038648869) roll requested 2024-11-19T17:50:48,904 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038648904 2024-11-19T17:50:48,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,930 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,930 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,931 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648869 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648904 2024-11-19T17:50:48,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:48,932 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,932 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038648904) roll requested 2024-11-19T17:50:48,932 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038648932 2024-11-19T17:50:48,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,952 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,953 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,953 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,953 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,953 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648904 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648932 2024-11-19T17:50:48,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:48,960 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:48,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038648932) roll requested 2024-11-19T17:50:48,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038648960 2024-11-19T17:50:48,988 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,988 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,989 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,989 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:48,989 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648932 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648960 2024-11-19T17:50:49,000 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,000 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,001 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038648960) roll requested 2024-11-19T17:50:49,001 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649001 2024-11-19T17:50:49,028 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,028 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,029 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,029 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,029 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038648960 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649001 2024-11-19T17:50:49,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,040 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,040 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649001) roll requested 2024-11-19T17:50:49,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649040 2024-11-19T17:50:49,048 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,048 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,049 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,049 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,049 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,049 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649001 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649040 2024-11-19T17:50:49,050 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,050 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,050 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,050 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649040) roll requested 2024-11-19T17:50:49,051 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649050 2024-11-19T17:50:49,068 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,068 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,068 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,068 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,068 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649040 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649050 2024-11-19T17:50:49,069 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,069 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,069 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649050) roll requested 2024-11-19T17:50:49,069 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649069 2024-11-19T17:50:49,087 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,087 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,087 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,087 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,087 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,087 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649050 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649069 2024-11-19T17:50:49,088 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,088 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,088 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,088 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649069) roll requested 2024-11-19T17:50:49,088 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649088 2024-11-19T17:50:49,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,097 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,097 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,097 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,097 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649069 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649088 2024-11-19T17:50:49,103 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,103 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,103 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,103 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649088) roll requested 2024-11-19T17:50:49,103 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649103 2024-11-19T17:50:49,110 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,110 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,110 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,110 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,111 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,111 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649088 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649103 2024-11-19T17:50:49,111 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,111 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,111 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,111 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649103) roll requested 2024-11-19T17:50:49,112 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649111 2024-11-19T17:50:49,130 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,131 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,131 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,131 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,131 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,131 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649103 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649111 2024-11-19T17:50:49,132 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,132 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,132 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,132 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649111) roll requested 2024-11-19T17:50:49,132 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649132 2024-11-19T17:50:49,146 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649111 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649132 2024-11-19T17:50:49,147 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,147 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,147 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649132) roll requested 2024-11-19T17:50:49,148 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649148 2024-11-19T17:50:49,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,160 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,160 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649132 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649148 2024-11-19T17:50:49,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,161 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649148) roll requested 2024-11-19T17:50:49,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649161 2024-11-19T17:50:49,168 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,168 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,168 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,168 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,169 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,169 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649148 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649161 2024-11-19T17:50:49,169 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,169 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,169 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,169 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649161) roll requested 2024-11-19T17:50:49,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649169 2024-11-19T17:50:49,184 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,185 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,185 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,185 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,185 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649161 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649169 2024-11-19T17:50:49,185 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,185 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,186 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,186 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649169) roll requested 2024-11-19T17:50:49,186 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649186 2024-11-19T17:50:49,193 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,193 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,193 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,194 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,194 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649169 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649186 2024-11-19T17:50:49,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,204 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,204 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649186) roll requested 2024-11-19T17:50:49,204 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649204 2024-11-19T17:50:49,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649186 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649204 2024-11-19T17:50:49,211 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,211 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,211 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649204) roll requested 2024-11-19T17:50:49,211 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649211 2024-11-19T17:50:49,218 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,219 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,219 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,219 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649204 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649211 2024-11-19T17:50:49,219 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,219 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,220 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,220 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649211) roll requested 2024-11-19T17:50:49,220 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649220 2024-11-19T17:50:49,226 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,226 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,226 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,226 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,226 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,226 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649211 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649220 2024-11-19T17:50:49,228 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,228 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,228 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,228 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649220) roll requested 2024-11-19T17:50:49,228 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649228 2024-11-19T17:50:49,246 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,247 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,247 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,247 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,247 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,247 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649220 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649228 2024-11-19T17:50:49,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,256 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,256 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649228) roll requested 2024-11-19T17:50:49,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649257 2024-11-19T17:50:49,272 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,273 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,273 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,273 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,273 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,273 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649228 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649257 2024-11-19T17:50:49,282 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,282 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,282 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,282 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649257) roll requested 2024-11-19T17:50:49,282 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649282 2024-11-19T17:50:49,296 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,296 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,296 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,296 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,297 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,297 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649257 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649282 2024-11-19T17:50:49,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,308 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,309 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649282) roll requested 2024-11-19T17:50:49,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649309 2024-11-19T17:50:49,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,319 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649282 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649309 2024-11-19T17:50:49,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,324 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649309) roll requested 2024-11-19T17:50:49,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649324 2024-11-19T17:50:49,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,339 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,339 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,340 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649309 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649324 2024-11-19T17:50:49,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,340 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,341 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649324) roll requested 2024-11-19T17:50:49,341 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649341 2024-11-19T17:50:49,352 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,352 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,352 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,353 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649324 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649341 2024-11-19T17:50:49,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,356 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649341) roll requested 2024-11-19T17:50:49,357 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649356 2024-11-19T17:50:49,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,376 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649341 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649356 2024-11-19T17:50:49,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,377 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,377 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649356) roll requested 2024-11-19T17:50:49,377 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649377 2024-11-19T17:50:49,382 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,382 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,382 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,382 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,382 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,382 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649356 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649377 2024-11-19T17:50:49,383 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,383 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,383 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,383 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649377) roll requested 2024-11-19T17:50:49,383 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649383 2024-11-19T17:50:49,400 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,400 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,400 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,400 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,400 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,400 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649377 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649383 2024-11-19T17:50:49,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,420 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649383) roll requested 2024-11-19T17:50:49,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649420 2024-11-19T17:50:49,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,426 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,426 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649383 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649420 2024-11-19T17:50:49,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,428 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649420) roll requested 2024-11-19T17:50:49,429 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649428 2024-11-19T17:50:49,435 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,435 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,435 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,435 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,435 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,435 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649420 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649428 2024-11-19T17:50:49,436 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,436 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,436 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,436 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649428) roll requested 2024-11-19T17:50:49,436 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649436 2024-11-19T17:50:49,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649428 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649436 2024-11-19T17:50:49,445 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,445 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,445 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649436) roll requested 2024-11-19T17:50:49,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649445 2024-11-19T17:50:49,454 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,454 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,454 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,454 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,454 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,455 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649436 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649445 2024-11-19T17:50:49,455 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,455 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,455 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,455 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649445) roll requested 2024-11-19T17:50:49,455 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649455 2024-11-19T17:50:49,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649445 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649455 2024-11-19T17:50:49,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,464 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649455) roll requested 2024-11-19T17:50:49,464 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649464 2024-11-19T17:50:49,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,474 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,474 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,474 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,474 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649455 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649464 2024-11-19T17:50:49,475 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,475 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,475 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649464) roll requested 2024-11-19T17:50:49,475 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649475 2024-11-19T17:50:49,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,481 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,481 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,481 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,481 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649464 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649475 2024-11-19T17:50:49,482 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,482 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649475) roll requested 2024-11-19T17:50:49,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649483 2024-11-19T17:50:49,491 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,491 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,491 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,491 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,492 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649475 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649483 2024-11-19T17:50:49,493 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,493 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649483) roll requested 2024-11-19T17:50:49,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649493 2024-11-19T17:50:49,499 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649483 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649493 2024-11-19T17:50:49,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,500 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649493) roll requested 2024-11-19T17:50:49,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649501 2024-11-19T17:50:49,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,506 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649493 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649501 2024-11-19T17:50:49,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,512 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,512 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649501) roll requested 2024-11-19T17:50:49,512 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649512 2024-11-19T17:50:49,518 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,518 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,518 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,518 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649501 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649512 2024-11-19T17:50:49,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,521 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649512) roll requested 2024-11-19T17:50:49,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649521 2024-11-19T17:50:49,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,531 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649512 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649521 2024-11-19T17:50:49,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,535 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,535 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649521) roll requested 2024-11-19T17:50:49,535 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649535 2024-11-19T17:50:49,552 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,552 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,552 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,552 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,552 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,552 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649521 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649535 2024-11-19T17:50:49,557 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,557 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649535) roll requested 2024-11-19T17:50:49,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649557 2024-11-19T17:50:49,567 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,567 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,567 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,568 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,568 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,568 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649535 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649557 2024-11-19T17:50:49,575 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,575 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,575 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649557) roll requested 2024-11-19T17:50:49,576 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649576 2024-11-19T17:50:49,581 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,582 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,582 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,582 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,582 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,582 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649557 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649576 2024-11-19T17:50:49,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,584 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649576) roll requested 2024-11-19T17:50:49,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649585 2024-11-19T17:50:49,604 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,604 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,604 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,604 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,604 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,605 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649576 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649585 2024-11-19T17:50:49,605 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,605 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,605 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,605 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649585) roll requested 2024-11-19T17:50:49,606 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649605 2024-11-19T17:50:49,620 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,620 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,620 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,620 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,621 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,621 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649585 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649605 2024-11-19T17:50:49,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,624 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649605) roll requested 2024-11-19T17:50:49,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649624 2024-11-19T17:50:49,631 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,632 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,632 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,632 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,632 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,632 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649605 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649624 2024-11-19T17:50:49,634 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,634 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,634 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,634 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649624) roll requested 2024-11-19T17:50:49,635 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649635 2024-11-19T17:50:49,643 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,644 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,644 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,644 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,644 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,644 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649624 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649635 2024-11-19T17:50:49,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649624 is not closed yet, will try archiving it next time 2024-11-19T17:50:49,644 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,645 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649635) roll requested 2024-11-19T17:50:49,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649645 2024-11-19T17:50:49,651 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,652 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,652 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,652 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,652 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649635 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649645 2024-11-19T17:50:49,653 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,653 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,653 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,653 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649645) roll requested 2024-11-19T17:50:49,653 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649653 2024-11-19T17:50:49,662 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,663 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649645 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649653 2024-11-19T17:50:49,668 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,668 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,668 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,668 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649653) roll requested 2024-11-19T17:50:49,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649668 2024-11-19T17:50:49,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,674 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,674 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,674 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649653 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649668 2024-11-19T17:50:49,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,676 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649668) roll requested 2024-11-19T17:50:49,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649676 2024-11-19T17:50:49,683 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,683 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,683 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,683 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,683 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,683 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649668 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649676 2024-11-19T17:50:49,692 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,692 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,692 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,692 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649676) roll requested 2024-11-19T17:50:49,693 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649692 2024-11-19T17:50:49,711 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,711 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,711 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,711 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,711 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649676 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649692 2024-11-19T17:50:49,717 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,717 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,717 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,717 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649692) roll requested 2024-11-19T17:50:49,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649717 2024-11-19T17:50:49,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,726 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,726 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,726 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,726 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649692 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649717 2024-11-19T17:50:49,728 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,728 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,728 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,728 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649717) roll requested 2024-11-19T17:50:49,729 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649728 2024-11-19T17:50:49,746 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,746 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,746 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,746 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,746 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,747 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649717 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649728 2024-11-19T17:50:49,747 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,747 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,747 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,747 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649728) roll requested 2024-11-19T17:50:49,748 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649748 2024-11-19T17:50:49,753 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,754 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,754 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,754 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,754 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,754 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649728 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649748 2024-11-19T17:50:49,754 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,754 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,754 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,754 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649748) roll requested 2024-11-19T17:50:49,755 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649754 2024-11-19T17:50:49,760 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,760 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,760 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,761 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,761 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,761 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649748 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649754 2024-11-19T17:50:49,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,764 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649754) roll requested 2024-11-19T17:50:49,765 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649764 2024-11-19T17:50:49,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,772 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,772 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649754 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649764 2024-11-19T17:50:49,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,776 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649764) roll requested 2024-11-19T17:50:49,776 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649776 2024-11-19T17:50:49,785 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,785 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,786 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,786 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,786 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,786 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649764 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649776 2024-11-19T17:50:49,786 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,786 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,786 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,786 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649776) roll requested 2024-11-19T17:50:49,787 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649787 2024-11-19T17:50:49,794 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,794 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,794 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,795 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649776 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649787 2024-11-19T17:50:49,795 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,795 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,795 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649787) roll requested 2024-11-19T17:50:49,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649796 2024-11-19T17:50:49,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,802 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,802 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649787 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649796 2024-11-19T17:50:49,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,804 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649796) roll requested 2024-11-19T17:50:49,804 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649804 2024-11-19T17:50:49,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649796 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649804 2024-11-19T17:50:49,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,812 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649804) roll requested 2024-11-19T17:50:49,812 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649812 2024-11-19T17:50:49,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,820 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649804 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649812 2024-11-19T17:50:49,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,821 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,822 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649812) roll requested 2024-11-19T17:50:49,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649822 2024-11-19T17:50:49,831 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,831 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649812 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649822 2024-11-19T17:50:49,832 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,832 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,832 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,832 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649822) roll requested 2024-11-19T17:50:49,832 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649832 2024-11-19T17:50:49,845 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,846 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649822 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649832 2024-11-19T17:50:49,846 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,846 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649832) roll requested 2024-11-19T17:50:49,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649847 2024-11-19T17:50:49,854 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,854 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,854 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,858 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649832 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649847 2024-11-19T17:50:49,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,864 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,864 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649847) roll requested 2024-11-19T17:50:49,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649865 2024-11-19T17:50:49,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,884 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649847 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649865 2024-11-19T17:50:49,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,887 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,887 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649865) roll requested 2024-11-19T17:50:49,887 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649887 2024-11-19T17:50:49,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,900 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,909 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,909 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,909 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,909 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649865 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649887 2024-11-19T17:50:49,909 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,910 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,910 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,910 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649887) roll requested 2024-11-19T17:50:49,910 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649910 2024-11-19T17:50:49,938 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,939 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,939 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,939 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649887 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649910 2024-11-19T17:50:49,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,940 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649910) roll requested 2024-11-19T17:50:49,940 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649940 2024-11-19T17:50:49,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,952 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,952 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,953 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,953 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,953 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649910 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649940 2024-11-19T17:50:49,957 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,957 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,957 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,957 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649940) roll requested 2024-11-19T17:50:49,957 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649957 2024-11-19T17:50:49,962 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,962 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,962 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,962 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,962 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,962 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649940 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649957 2024-11-19T17:50:49,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,968 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,968 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649957) roll requested 2024-11-19T17:50:49,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649968 2024-11-19T17:50:49,974 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,975 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,975 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,975 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,975 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,975 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649957 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649968 2024-11-19T17:50:49,985 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,985 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,985 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649968) roll requested 2024-11-19T17:50:49,986 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649985 2024-11-19T17:50:49,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,992 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,992 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:49,992 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649968 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649985 2024-11-19T17:50:49,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:49,996 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,996 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:49,996 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649985) roll requested 2024-11-19T17:50:49,996 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038649996 2024-11-19T17:50:50,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,010 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649985 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649996 2024-11-19T17:50:50,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,016 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,016 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038649996) roll requested 2024-11-19T17:50:50,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650016 2024-11-19T17:50:50,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,023 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,023 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038649996 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650016 2024-11-19T17:50:50,023 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,023 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,023 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,023 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650016) roll requested 2024-11-19T17:50:50,024 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650023 2024-11-19T17:50:50,034 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,035 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,035 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,035 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650016 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650023 2024-11-19T17:50:50,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,036 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,036 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,036 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650023) roll requested 2024-11-19T17:50:50,036 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650036 2024-11-19T17:50:50,046 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,046 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650023 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650036 2024-11-19T17:50:50,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,056 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,057 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650036) roll requested 2024-11-19T17:50:50,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650057 2024-11-19T17:50:50,065 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,065 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,065 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,065 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,066 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,066 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650036 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650057 2024-11-19T17:50:50,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,072 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,072 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,072 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650057) roll requested 2024-11-19T17:50:50,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650073 2024-11-19T17:50:50,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650057 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650073 2024-11-19T17:50:50,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,096 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650073) roll requested 2024-11-19T17:50:50,096 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650096 2024-11-19T17:50:50,101 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,101 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,102 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,102 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,102 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,102 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650073 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650096 2024-11-19T17:50:50,103 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,103 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,103 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,103 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650096) roll requested 2024-11-19T17:50:50,103 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650103 2024-11-19T17:50:50,108 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,108 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,109 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,109 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,109 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650096 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650103 2024-11-19T17:50:50,112 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,112 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,112 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650103) roll requested 2024-11-19T17:50:50,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650112 2024-11-19T17:50:50,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650103 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650112 2024-11-19T17:50:50,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,120 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650112) roll requested 2024-11-19T17:50:50,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650120 2024-11-19T17:50:50,134 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,134 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,135 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,135 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,135 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650112 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650120 2024-11-19T17:50:50,136 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,136 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,137 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,137 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650120) roll requested 2024-11-19T17:50:50,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650137 2024-11-19T17:50:50,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,143 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,143 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650120 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650137 2024-11-19T17:50:50,144 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,144 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,144 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,144 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650137) roll requested 2024-11-19T17:50:50,144 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650144 2024-11-19T17:50:50,154 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,154 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,154 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650137 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650144 2024-11-19T17:50:50,155 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,155 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,155 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,155 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650144) roll requested 2024-11-19T17:50:50,156 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650156 2024-11-19T17:50:50,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,160 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650144 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650156 2024-11-19T17:50:50,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,164 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,164 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,164 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650156) roll requested 2024-11-19T17:50:50,164 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650164 2024-11-19T17:50:50,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,170 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,170 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,170 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,170 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650156 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650164 2024-11-19T17:50:50,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,178 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650164) roll requested 2024-11-19T17:50:50,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650179 2024-11-19T17:50:50,185 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,185 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,185 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,185 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,185 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650164 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650179 2024-11-19T17:50:50,192 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,192 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,192 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,192 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650179) roll requested 2024-11-19T17:50:50,193 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650192 2024-11-19T17:50:50,198 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,198 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,199 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,199 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,200 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,200 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650179 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650192 2024-11-19T17:50:50,200 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,200 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,200 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650192) roll requested 2024-11-19T17:50:50,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650201 2024-11-19T17:50:50,208 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,208 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,209 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650192 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650201 2024-11-19T17:50:50,212 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,212 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,212 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,212 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650201) roll requested 2024-11-19T17:50:50,213 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650213 2024-11-19T17:50:50,220 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,220 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,220 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,220 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,221 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,221 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650201 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650213 2024-11-19T17:50:50,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,224 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,224 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,224 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650213) roll requested 2024-11-19T17:50:50,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650224 2024-11-19T17:50:50,231 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,231 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,232 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,232 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,232 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650213 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650224 2024-11-19T17:50:50,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,233 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,234 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,234 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650224) roll requested 2024-11-19T17:50:50,234 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650234 2024-11-19T17:50:50,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,240 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,240 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,241 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,241 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650224 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650234 2024-11-19T17:50:50,241 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,241 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,241 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,242 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650234) roll requested 2024-11-19T17:50:50,242 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650242 2024-11-19T17:50:50,251 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,251 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,251 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,252 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650234 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650242 2024-11-19T17:50:50,253 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,253 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,253 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650242) roll requested 2024-11-19T17:50:50,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650253 2024-11-19T17:50:50,268 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,268 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,268 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,268 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,268 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,269 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650242 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650253 2024-11-19T17:50:50,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,272 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,273 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650253) roll requested 2024-11-19T17:50:50,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650273 2024-11-19T17:50:50,279 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,279 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,279 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,279 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,279 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,279 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650253 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650273 2024-11-19T17:50:50,280 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,280 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,280 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,280 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650273) roll requested 2024-11-19T17:50:50,281 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650280 2024-11-19T17:50:50,294 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,294 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,294 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,295 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,295 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650273 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650280 2024-11-19T17:50:50,296 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,296 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,296 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,297 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650280) roll requested 2024-11-19T17:50:50,297 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650297 2024-11-19T17:50:50,302 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,302 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,302 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,303 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,303 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650280 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650297 2024-11-19T17:50:50,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,304 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650297) roll requested 2024-11-19T17:50:50,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650304 2024-11-19T17:50:50,310 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,310 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,310 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,310 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,310 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,310 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650297 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650304 2024-11-19T17:50:50,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,316 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,316 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650304) roll requested 2024-11-19T17:50:50,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650317 2024-11-19T17:50:50,322 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,322 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,322 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,322 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,323 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,323 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650304 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650317 2024-11-19T17:50:50,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,324 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650317) roll requested 2024-11-19T17:50:50,324 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650324 2024-11-19T17:50:50,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,331 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,331 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,332 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,332 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650317 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650324 2024-11-19T17:50:50,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,334 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,335 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,335 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650324) roll requested 2024-11-19T17:50:50,335 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650335 2024-11-19T17:50:50,340 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,340 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,341 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,344 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,344 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,344 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650324 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650335 2024-11-19T17:50:50,345 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,345 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,345 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,345 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650335) roll requested 2024-11-19T17:50:50,345 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650345 2024-11-19T17:50:50,351 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,352 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,352 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,352 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,352 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650335 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650345 2024-11-19T17:50:50,353 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,353 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,354 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,354 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650345) roll requested 2024-11-19T17:50:50,354 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650354 2024-11-19T17:50:50,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,360 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,360 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650345 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650354 2024-11-19T17:50:50,383 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,383 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650354) roll requested 2024-11-19T17:50:50,384 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650384 2024-11-19T17:50:50,409 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,409 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,409 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,409 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,409 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,409 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650354 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650384 2024-11-19T17:50:50,416 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,416 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,417 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,418 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650384) roll requested 2024-11-19T17:50:50,418 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650418 2024-11-19T17:50:50,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650384 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650418 2024-11-19T17:50:50,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,432 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650418) roll requested 2024-11-19T17:50:50,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650433 2024-11-19T17:50:50,448 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,448 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,448 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,449 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650418 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650433 2024-11-19T17:50:50,450 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,450 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650433) roll requested 2024-11-19T17:50:50,451 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650451 2024-11-19T17:50:50,472 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,476 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,476 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,476 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,476 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650433 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650451 2024-11-19T17:50:50,478 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,478 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650451) roll requested 2024-11-19T17:50:50,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650478 2024-11-19T17:50:50,496 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,496 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,496 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,496 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,497 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,497 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650451 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650478 2024-11-19T17:50:50,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,500 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,500 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650478) roll requested 2024-11-19T17:50:50,500 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650500 2024-11-19T17:50:50,512 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,512 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,512 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650478 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650500 2024-11-19T17:50:50,517 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,517 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,517 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,517 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650500) roll requested 2024-11-19T17:50:50,517 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650517 2024-11-19T17:50:50,522 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,522 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,523 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,523 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650500 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650517 2024-11-19T17:50:50,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,531 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650517) roll requested 2024-11-19T17:50:50,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650531 2024-11-19T17:50:50,536 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,537 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,537 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,537 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,544 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,544 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650517 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650531 2024-11-19T17:50:50,545 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,546 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,546 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,546 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650531) roll requested 2024-11-19T17:50:50,546 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650546 2024-11-19T17:50:50,558 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,558 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,558 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,558 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,558 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,558 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650531 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650546 2024-11-19T17:50:50,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,559 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,559 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,559 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650546) roll requested 2024-11-19T17:50:50,559 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650559 2024-11-19T17:50:50,572 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,572 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,572 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,572 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,573 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,573 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650546 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650559 2024-11-19T17:50:50,573 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,573 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,573 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,573 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650559) roll requested 2024-11-19T17:50:50,573 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650573 2024-11-19T17:50:50,579 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,579 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,579 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650559 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650573 2024-11-19T17:50:50,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,584 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650573) roll requested 2024-11-19T17:50:50,584 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650584 2024-11-19T17:50:50,590 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,591 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650573 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650584 2024-11-19T17:50:50,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,593 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650584) roll requested 2024-11-19T17:50:50,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650593 2024-11-19T17:50:50,604 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,605 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,605 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,605 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,605 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,605 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650584 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650593 2024-11-19T17:50:50,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,608 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650593) roll requested 2024-11-19T17:50:50,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650609 2024-11-19T17:50:50,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,619 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,619 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,619 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,619 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,619 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650593 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650609 2024-11-19T17:50:50,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,620 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650609) roll requested 2024-11-19T17:50:50,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650621 2024-11-19T17:50:50,637 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,637 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,637 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,638 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650609 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650621 2024-11-19T17:50:50,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,640 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,640 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650621) roll requested 2024-11-19T17:50:50,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650641 2024-11-19T17:50:50,649 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,649 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,650 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,650 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,650 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650621 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650641 2024-11-19T17:50:50,652 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,655 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650641) roll requested 2024-11-19T17:50:50,656 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650656 2024-11-19T17:50:50,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,667 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,667 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,667 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650641 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650656 2024-11-19T17:50:50,681 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,681 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,681 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650656) roll requested 2024-11-19T17:50:50,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650681 2024-11-19T17:50:50,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650656 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650681 2024-11-19T17:50:50,699 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,699 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,699 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,699 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650681) roll requested 2024-11-19T17:50:50,699 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650699 2024-11-19T17:50:50,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,709 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,709 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650681 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650699 2024-11-19T17:50:50,709 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,709 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650699) roll requested 2024-11-19T17:50:50,710 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650710 2024-11-19T17:50:50,723 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,723 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,723 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,723 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,723 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,723 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650699 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650710 2024-11-19T17:50:50,725 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,725 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,725 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650710) roll requested 2024-11-19T17:50:50,726 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650725 2024-11-19T17:50:50,734 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650710 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650725 2024-11-19T17:50:50,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,736 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,736 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650725) roll requested 2024-11-19T17:50:50,737 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650736 2024-11-19T17:50:50,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,744 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,744 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,744 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650725 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650736 2024-11-19T17:50:50,745 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,745 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,745 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650736) roll requested 2024-11-19T17:50:50,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650745 2024-11-19T17:50:50,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,754 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,754 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,754 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650736 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650745 2024-11-19T17:50:50,755 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,755 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,755 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,755 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650745) roll requested 2024-11-19T17:50:50,755 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650755 2024-11-19T17:50:50,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650745 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650755 2024-11-19T17:50:50,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,772 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650755) roll requested 2024-11-19T17:50:50,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650773 2024-11-19T17:50:50,780 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,781 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,781 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650755 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650773 2024-11-19T17:50:50,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,796 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650773) roll requested 2024-11-19T17:50:50,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650796 2024-11-19T17:50:50,804 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,804 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650773 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650796 2024-11-19T17:50:50,807 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,812 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650796) roll requested 2024-11-19T17:50:50,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650813 2024-11-19T17:50:50,828 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,828 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,828 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,828 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,830 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650796 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650813 2024-11-19T17:50:50,830 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,830 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,830 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,830 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650813) roll requested 2024-11-19T17:50:50,831 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650830 2024-11-19T17:50:50,844 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,844 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,844 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,844 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,844 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,844 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650813 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650830 2024-11-19T17:50:50,846 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,846 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650830) roll requested 2024-11-19T17:50:50,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650847 2024-11-19T17:50:50,854 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,854 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650830 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650847 2024-11-19T17:50:50,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,857 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650847) roll requested 2024-11-19T17:50:50,857 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650857 2024-11-19T17:50:50,864 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,864 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,864 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650847 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650857 2024-11-19T17:50:50,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,867 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,867 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650857) roll requested 2024-11-19T17:50:50,867 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650867 2024-11-19T17:50:50,876 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,876 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,876 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,876 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,876 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650857 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650867 2024-11-19T17:50:50,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,880 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,880 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,880 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650867) roll requested 2024-11-19T17:50:50,880 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650880 2024-11-19T17:50:50,889 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,889 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,889 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650867 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650880 2024-11-19T17:50:50,892 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,892 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,892 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650880) roll requested 2024-11-19T17:50:50,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650893 2024-11-19T17:50:50,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650880 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650893 2024-11-19T17:50:50,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,900 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650893) roll requested 2024-11-19T17:50:50,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650900 2024-11-19T17:50:50,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650893 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650900 2024-11-19T17:50:50,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,908 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,908 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650900) roll requested 2024-11-19T17:50:50,908 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650908 2024-11-19T17:50:50,915 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,915 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,915 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,915 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,915 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,915 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650900 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650908 2024-11-19T17:50:50,916 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,916 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,916 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,916 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650908) roll requested 2024-11-19T17:50:50,916 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650916 2024-11-19T17:50:50,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,926 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,926 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,926 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650908 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650916 2024-11-19T17:50:50,928 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,928 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,928 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650916) roll requested 2024-11-19T17:50:50,929 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650928 2024-11-19T17:50:50,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,934 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650916 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650928 2024-11-19T17:50:50,936 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,936 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,936 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,936 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650928) roll requested 2024-11-19T17:50:50,936 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650936 2024-11-19T17:50:50,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,945 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:50,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650928 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650936 2024-11-19T17:50:50,953 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:50,954 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,954 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:50,954 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650936) roll requested 2024-11-19T17:50:50,954 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038650954 2024-11-19T17:50:51,005 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,005 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,005 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650936 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650954 2024-11-19T17:50:51,009 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,009 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038650954) roll requested 2024-11-19T17:50:51,009 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651009 2024-11-19T17:50:51,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,019 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,019 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038650954 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651009 2024-11-19T17:50:51,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,020 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,020 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,021 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651009) roll requested 2024-11-19T17:50:51,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651021 2024-11-19T17:50:51,028 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,029 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,029 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,029 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,029 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651009 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651021 2024-11-19T17:50:51,030 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,030 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,030 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,030 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651021) roll requested 2024-11-19T17:50:51,030 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651030 2024-11-19T17:50:51,053 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,053 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,054 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,054 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,055 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,055 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651021 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651030 2024-11-19T17:50:51,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,064 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,065 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,065 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651030) roll requested 2024-11-19T17:50:51,065 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651065 2024-11-19T17:50:51,075 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,075 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651030 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651065 2024-11-19T17:50:51,078 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,078 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,078 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,078 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651065) roll requested 2024-11-19T17:50:51,078 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651078 2024-11-19T17:50:51,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,088 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651065 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651078 2024-11-19T17:50:51,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,096 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651078) roll requested 2024-11-19T17:50:51,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651096 2024-11-19T17:50:51,107 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,107 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,107 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,107 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,107 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,108 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651078 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651096 2024-11-19T17:50:51,108 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,108 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,108 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,108 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651096) roll requested 2024-11-19T17:50:51,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651108 2024-11-19T17:50:51,120 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,121 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,121 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,121 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,121 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,121 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651096 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651108 2024-11-19T17:50:51,122 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,122 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,122 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,122 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651108) roll requested 2024-11-19T17:50:51,123 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651123 2024-11-19T17:50:51,131 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,131 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,131 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,131 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,131 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,131 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651108 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651123 2024-11-19T17:50:51,132 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,132 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,132 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,132 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651123) roll requested 2024-11-19T17:50:51,132 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651132 2024-11-19T17:50:51,146 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,146 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,146 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651123 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651132 2024-11-19T17:50:51,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,148 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651132) roll requested 2024-11-19T17:50:51,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651149 2024-11-19T17:50:51,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,161 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,161 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,161 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651132 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651149 2024-11-19T17:50:51,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,164 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,164 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651149) roll requested 2024-11-19T17:50:51,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651165 2024-11-19T17:50:51,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,177 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,177 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,177 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,177 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,177 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651149 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651165 2024-11-19T17:50:51,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,184 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,185 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651165) roll requested 2024-11-19T17:50:51,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651185 2024-11-19T17:50:51,191 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,191 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,191 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,191 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,191 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,191 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651165 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651185 2024-11-19T17:50:51,199 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,199 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,199 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,199 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651185) roll requested 2024-11-19T17:50:51,199 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651199 2024-11-19T17:50:51,208 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,208 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,208 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,209 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651185 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651199 2024-11-19T17:50:51,212 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,212 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,212 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,212 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651199) roll requested 2024-11-19T17:50:51,213 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651212 2024-11-19T17:50:51,219 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,219 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,219 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,219 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651199 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651212 2024-11-19T17:50:51,220 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,220 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,221 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,221 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651212) roll requested 2024-11-19T17:50:51,221 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651221 2024-11-19T17:50:51,227 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,227 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651212 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651221 2024-11-19T17:50:51,228 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,228 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,228 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,228 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651221) roll requested 2024-11-19T17:50:51,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651228 2024-11-19T17:50:51,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,240 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,240 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651221 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651228 2024-11-19T17:50:51,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,264 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651228) roll requested 2024-11-19T17:50:51,265 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651265 2024-11-19T17:50:51,291 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651228 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651265 2024-11-19T17:50:51,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,293 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651265) roll requested 2024-11-19T17:50:51,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651293 2024-11-19T17:50:51,304 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,305 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,305 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651265 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651293 2024-11-19T17:50:51,311 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,311 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,311 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,311 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651293) roll requested 2024-11-19T17:50:51,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651311 2024-11-19T17:50:51,320 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,320 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,320 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,320 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,320 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,321 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651293 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651311 2024-11-19T17:50:51,327 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,327 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,327 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651311) roll requested 2024-11-19T17:50:51,327 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651327 2024-11-19T17:50:51,335 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,335 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,335 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,335 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,335 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,335 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651311 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651327 2024-11-19T17:50:51,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,340 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,340 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651327) roll requested 2024-11-19T17:50:51,341 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651340 2024-11-19T17:50:51,350 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,351 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,351 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,351 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,351 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,351 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651327 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651340 2024-11-19T17:50:51,352 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,352 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,352 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,353 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651340) roll requested 2024-11-19T17:50:51,353 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651353 2024-11-19T17:50:51,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,371 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651340 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651353 2024-11-19T17:50:51,371 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,371 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,372 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651353) roll requested 2024-11-19T17:50:51,372 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651372 2024-11-19T17:50:51,379 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,379 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,379 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,379 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,379 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651353 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651372 2024-11-19T17:50:51,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,379 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,379 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651372) roll requested 2024-11-19T17:50:51,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651380 2024-11-19T17:50:51,386 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,386 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,386 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,386 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651372 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651380 2024-11-19T17:50:51,388 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,388 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,388 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,388 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651380) roll requested 2024-11-19T17:50:51,388 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651388 2024-11-19T17:50:51,395 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,395 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,395 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,395 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,395 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,395 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651380 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651388 2024-11-19T17:50:51,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,396 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,396 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651388) roll requested 2024-11-19T17:50:51,396 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651396 2024-11-19T17:50:51,409 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,409 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,409 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,409 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,409 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,409 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651388 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651396 2024-11-19T17:50:51,416 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,416 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,416 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,416 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651396) roll requested 2024-11-19T17:50:51,417 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651416 2024-11-19T17:50:51,423 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,423 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651396 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651416 2024-11-19T17:50:51,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,432 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,433 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651416) roll requested 2024-11-19T17:50:51,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651433 2024-11-19T17:50:51,442 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,442 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,444 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,444 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651416 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651433 2024-11-19T17:50:51,445 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,445 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,445 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651433) roll requested 2024-11-19T17:50:51,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651445 2024-11-19T17:50:51,454 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,454 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,454 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,454 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,454 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,455 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651433 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651445 2024-11-19T17:50:51,455 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,455 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651445) roll requested 2024-11-19T17:50:51,456 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651456 2024-11-19T17:50:51,472 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,472 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,472 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651445 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651456 2024-11-19T17:50:51,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,473 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651456) roll requested 2024-11-19T17:50:51,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651473 2024-11-19T17:50:51,478 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,478 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,478 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,479 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,479 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651456 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651473 2024-11-19T17:50:51,479 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,479 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,479 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,479 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651473) roll requested 2024-11-19T17:50:51,479 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651479 2024-11-19T17:50:51,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,495 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651473 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651479 2024-11-19T17:50:51,495 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,495 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651479) roll requested 2024-11-19T17:50:51,496 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651496 2024-11-19T17:50:51,508 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,508 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,508 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,508 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,509 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,509 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651479 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651496 2024-11-19T17:50:51,516 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,516 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,516 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,516 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651496) roll requested 2024-11-19T17:50:51,516 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651516 2024-11-19T17:50:51,522 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,522 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,522 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,522 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,522 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,522 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651496 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651516 2024-11-19T17:50:51,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,524 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,524 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,524 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651516) roll requested 2024-11-19T17:50:51,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651525 2024-11-19T17:50:51,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,535 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,535 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651516 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651525 2024-11-19T17:50:51,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,541 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,541 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651525) roll requested 2024-11-19T17:50:51,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651541 2024-11-19T17:50:51,550 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,550 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,550 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,550 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,550 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,551 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651525 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651541 2024-11-19T17:50:51,551 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,551 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,551 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,551 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651541) roll requested 2024-11-19T17:50:51,551 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651551 2024-11-19T17:50:51,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,562 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,562 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651541 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651551 2024-11-19T17:50:51,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,564 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,564 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651551) roll requested 2024-11-19T17:50:51,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651564 2024-11-19T17:50:51,572 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,573 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,573 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,573 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,573 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,573 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651551 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651564 2024-11-19T17:50:51,574 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,574 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,574 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,574 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651564) roll requested 2024-11-19T17:50:51,574 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651574 2024-11-19T17:50:51,579 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,579 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,579 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651564 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651574 2024-11-19T17:50:51,581 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,580 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,581 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651574) roll requested 2024-11-19T17:50:51,581 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651581 2024-11-19T17:50:51,587 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651574 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651581 2024-11-19T17:50:51,588 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,588 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,588 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,588 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651581) roll requested 2024-11-19T17:50:51,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651588 2024-11-19T17:50:51,596 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,596 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,596 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,596 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,596 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,596 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651581 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651588 2024-11-19T17:50:51,597 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,597 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,597 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,597 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651588) roll requested 2024-11-19T17:50:51,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651597 2024-11-19T17:50:51,606 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,606 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,606 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,606 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,606 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,606 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651588 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651597 2024-11-19T17:50:51,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,608 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651597) roll requested 2024-11-19T17:50:51,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651608 2024-11-19T17:50:51,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,619 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,619 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,620 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,620 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,620 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651597 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651608 2024-11-19T17:50:51,621 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,621 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,621 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651608) roll requested 2024-11-19T17:50:51,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651621 2024-11-19T17:50:51,631 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,631 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,632 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651608 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651621 2024-11-19T17:50:51,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,632 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,633 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651621) roll requested 2024-11-19T17:50:51,633 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651633 2024-11-19T17:50:51,638 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,638 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,639 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,639 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651621 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651633 2024-11-19T17:50:51,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,640 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,640 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651633) roll requested 2024-11-19T17:50:51,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651641 2024-11-19T17:50:51,647 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,647 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,647 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,647 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,647 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,647 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651633 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651641 2024-11-19T17:50:51,647 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,647 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,648 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651641) roll requested 2024-11-19T17:50:51,648 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651648 2024-11-19T17:50:51,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,657 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,657 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,657 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,657 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,657 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651641 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651648 2024-11-19T17:50:51,663 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,663 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651648) roll requested 2024-11-19T17:50:51,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651664 2024-11-19T17:50:51,670 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,670 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,670 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,670 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,670 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,670 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651648 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651664 2024-11-19T17:50:51,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,672 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,673 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651664) roll requested 2024-11-19T17:50:51,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651673 2024-11-19T17:50:51,683 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,683 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,684 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,684 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,684 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,684 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651664 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651673 2024-11-19T17:50:51,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,684 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,685 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,685 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651673) roll requested 2024-11-19T17:50:51,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651685 2024-11-19T17:50:51,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,695 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651673 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651685 2024-11-19T17:50:51,698 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,698 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,698 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,698 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651685) roll requested 2024-11-19T17:50:51,698 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651698 2024-11-19T17:50:51,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,713 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,713 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651685 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651698 2024-11-19T17:50:51,714 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,714 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,714 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651698) roll requested 2024-11-19T17:50:51,715 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651714 2024-11-19T17:50:51,728 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,728 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,728 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,728 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,728 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,729 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651698 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651714 2024-11-19T17:50:51,730 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,730 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,730 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,730 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651714) roll requested 2024-11-19T17:50:51,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651730 2024-11-19T17:50:51,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,744 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,744 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,744 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,744 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651714 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651730 2024-11-19T17:50:51,746 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,746 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,746 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651730) roll requested 2024-11-19T17:50:51,746 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651746 2024-11-19T17:50:51,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,762 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,762 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651730 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651746 2024-11-19T17:50:51,763 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,763 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651746) roll requested 2024-11-19T17:50:51,764 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651764 2024-11-19T17:50:51,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,773 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651746 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651764 2024-11-19T17:50:51,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,776 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651764) roll requested 2024-11-19T17:50:51,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651776 2024-11-19T17:50:51,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,782 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,783 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,784 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651764 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651776 2024-11-19T17:50:51,785 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,785 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,785 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651776) roll requested 2024-11-19T17:50:51,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651785 2024-11-19T17:50:51,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,792 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,792 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,792 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,792 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651776 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651785 2024-11-19T17:50:51,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,796 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,797 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651785) roll requested 2024-11-19T17:50:51,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651797 2024-11-19T17:50:51,808 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,808 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,808 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,809 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,809 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,809 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651785 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651797 2024-11-19T17:50:51,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,812 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651797) roll requested 2024-11-19T17:50:51,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651813 2024-11-19T17:50:51,824 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,825 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,825 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651797 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651813 2024-11-19T17:50:51,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,828 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651813) roll requested 2024-11-19T17:50:51,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651829 2024-11-19T17:50:51,849 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,850 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,850 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,850 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,850 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,850 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651813 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651829 2024-11-19T17:50:51,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,864 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,865 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651829) roll requested 2024-11-19T17:50:51,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651865 2024-11-19T17:50:51,875 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,876 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,876 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,876 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,876 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,876 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651829 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651865 2024-11-19T17:50:51,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,876 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,877 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,877 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651865) roll requested 2024-11-19T17:50:51,877 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651877 2024-11-19T17:50:51,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,900 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651865 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651877 2024-11-19T17:50:51,902 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,902 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,902 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,902 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651877) roll requested 2024-11-19T17:50:51,902 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651902 2024-11-19T17:50:51,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,919 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,919 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651877 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651902 2024-11-19T17:50:51,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,920 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651902) roll requested 2024-11-19T17:50:51,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651921 2024-11-19T17:50:51,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,934 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651902 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651921 2024-11-19T17:50:51,935 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,935 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,935 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,935 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651921) roll requested 2024-11-19T17:50:51,935 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651935 2024-11-19T17:50:51,948 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,948 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,948 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,948 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,948 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,948 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651921 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651935 2024-11-19T17:50:51,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,948 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651935) roll requested 2024-11-19T17:50:51,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651949 2024-11-19T17:50:51,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,959 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,959 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651935 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651949 2024-11-19T17:50:51,959 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,959 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,959 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,959 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651949) roll requested 2024-11-19T17:50:51,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651959 2024-11-19T17:50:51,964 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,964 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,964 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,964 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,965 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,965 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651949 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651959 2024-11-19T17:50:51,966 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,967 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,967 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,967 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651959) roll requested 2024-11-19T17:50:51,967 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651967 2024-11-19T17:50:51,972 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,972 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,973 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,973 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,973 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,973 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651959 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651967 2024-11-19T17:50:51,976 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,976 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,976 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,976 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651967) roll requested 2024-11-19T17:50:51,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651976 2024-11-19T17:50:51,983 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,983 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,983 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,984 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,984 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651967 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651976 2024-11-19T17:50:51,984 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:51,985 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:51,985 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651976) roll requested 2024-11-19T17:50:51,985 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038651985 2024-11-19T17:50:51,994 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,994 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,994 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,994 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,994 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:51,994 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651976 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651985 2024-11-19T17:50:52,005 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,005 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,005 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,006 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038651985) roll requested 2024-11-19T17:50:52,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652006 2024-11-19T17:50:52,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,017 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038651985 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652006 2024-11-19T17:50:52,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,021 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,021 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,021 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652006) roll requested 2024-11-19T17:50:52,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652021 2024-11-19T17:50:52,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652006 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652021 2024-11-19T17:50:52,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,033 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,033 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652021) roll requested 2024-11-19T17:50:52,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652033 2024-11-19T17:50:52,043 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,043 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,043 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,044 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,044 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,044 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652021 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652033 2024-11-19T17:50:52,045 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,045 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,045 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652033) roll requested 2024-11-19T17:50:52,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652045 2024-11-19T17:50:52,054 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,054 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,054 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,054 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,055 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,055 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652033 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652045 2024-11-19T17:50:52,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,056 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652045) roll requested 2024-11-19T17:50:52,056 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652056 2024-11-19T17:50:52,068 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652045 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652056 2024-11-19T17:50:52,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,073 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652056) roll requested 2024-11-19T17:50:52,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652073 2024-11-19T17:50:52,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,091 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,091 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652056 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652073 2024-11-19T17:50:52,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,092 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,092 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652073) roll requested 2024-11-19T17:50:52,092 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652092 2024-11-19T17:50:52,107 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,107 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,107 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,107 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,107 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,108 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652073 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652092 2024-11-19T17:50:52,108 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,108 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,108 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,109 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652092) roll requested 2024-11-19T17:50:52,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652109 2024-11-19T17:50:52,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652092 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652109 2024-11-19T17:50:52,124 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,125 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,125 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,125 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652109) roll requested 2024-11-19T17:50:52,125 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652125 2024-11-19T17:50:52,131 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,131 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,131 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,132 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,132 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,132 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652109 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652125 2024-11-19T17:50:52,140 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,140 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,140 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,140 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652125) roll requested 2024-11-19T17:50:52,140 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652140 2024-11-19T17:50:52,146 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,146 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,146 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652125 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652140 2024-11-19T17:50:52,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,148 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652140) roll requested 2024-11-19T17:50:52,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652149 2024-11-19T17:50:52,162 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,162 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,163 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,163 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,163 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652140 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652149 2024-11-19T17:50:52,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,165 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652149) roll requested 2024-11-19T17:50:52,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652165 2024-11-19T17:50:52,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652149 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652165 2024-11-19T17:50:52,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,176 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,176 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652165) roll requested 2024-11-19T17:50:52,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652176 2024-11-19T17:50:52,191 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,191 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,191 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,191 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,191 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,192 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652165 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652176 2024-11-19T17:50:52,192 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,192 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,192 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,192 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652176) roll requested 2024-11-19T17:50:52,193 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652193 2024-11-19T17:50:52,203 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,203 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652176 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652193 2024-11-19T17:50:52,205 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,205 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,205 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652193) roll requested 2024-11-19T17:50:52,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652205 2024-11-19T17:50:52,216 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,216 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,216 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,217 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,217 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,217 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652193 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652205 2024-11-19T17:50:52,218 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,218 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,218 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652205) roll requested 2024-11-19T17:50:52,219 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652219 2024-11-19T17:50:52,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652205 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652219 2024-11-19T17:50:52,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,240 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,240 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652219) roll requested 2024-11-19T17:50:52,240 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652240 2024-11-19T17:50:52,247 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,247 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,247 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,247 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,247 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,247 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652219 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652240 2024-11-19T17:50:52,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,251 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,251 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652240) roll requested 2024-11-19T17:50:52,251 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652251 2024-11-19T17:50:52,260 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,260 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,261 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,261 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,261 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,261 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652240 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652251 2024-11-19T17:50:52,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,264 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652251) roll requested 2024-11-19T17:50:52,264 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652264 2024-11-19T17:50:52,270 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,270 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,271 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,271 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652251 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652264 2024-11-19T17:50:52,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,272 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652264) roll requested 2024-11-19T17:50:52,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652273 2024-11-19T17:50:52,279 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,279 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,279 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,279 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,280 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,280 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652264 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652273 2024-11-19T17:50:52,281 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,281 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,281 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,281 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652273) roll requested 2024-11-19T17:50:52,282 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652281 2024-11-19T17:50:52,302 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,302 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,302 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,302 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,303 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652273 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652281 2024-11-19T17:50:52,303 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,303 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,303 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,303 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652281) roll requested 2024-11-19T17:50:52,303 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652303 2024-11-19T17:50:52,314 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652281 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652303 2024-11-19T17:50:52,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,320 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,321 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652303) roll requested 2024-11-19T17:50:52,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652321 2024-11-19T17:50:52,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,331 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,331 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,331 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,331 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652303 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652321 2024-11-19T17:50:52,331 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,331 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,332 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652321) roll requested 2024-11-19T17:50:52,332 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652332 2024-11-19T17:50:52,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,339 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,339 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,339 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,340 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652321 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652332 2024-11-19T17:50:52,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,341 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,341 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,341 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652332) roll requested 2024-11-19T17:50:52,341 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652341 2024-11-19T17:50:52,355 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,355 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,355 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652332 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652341 2024-11-19T17:50:52,355 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,355 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652341) roll requested 2024-11-19T17:50:52,356 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652356 2024-11-19T17:50:52,379 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,379 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,379 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,379 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,379 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652341 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652356 2024-11-19T17:50:52,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,380 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652356) roll requested 2024-11-19T17:50:52,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652380 2024-11-19T17:50:52,386 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,387 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,387 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,387 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,387 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,388 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652356 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652380 2024-11-19T17:50:52,388 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,388 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,388 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652380) roll requested 2024-11-19T17:50:52,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652389 2024-11-19T17:50:52,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,396 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,396 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,397 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652380 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652389 2024-11-19T17:50:52,397 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,397 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652389) roll requested 2024-11-19T17:50:52,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652397 2024-11-19T17:50:52,409 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,409 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,409 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,409 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,409 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,409 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652389 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652397 2024-11-19T17:50:52,409 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,409 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,410 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,410 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652397) roll requested 2024-11-19T17:50:52,410 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652410 2024-11-19T17:50:52,422 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,422 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652397 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652410 2024-11-19T17:50:52,423 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,423 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,423 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652410) roll requested 2024-11-19T17:50:52,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652424 2024-11-19T17:50:52,434 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,435 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,435 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,435 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,435 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,435 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652410 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652424 2024-11-19T17:50:52,436 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,436 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,436 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,436 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652424) roll requested 2024-11-19T17:50:52,436 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652436 2024-11-19T17:50:52,447 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,447 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,447 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,447 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652424 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652436 2024-11-19T17:50:52,448 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,448 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,448 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,448 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652436) roll requested 2024-11-19T17:50:52,448 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652448 2024-11-19T17:50:52,457 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,457 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,457 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,457 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652436 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652448 2024-11-19T17:50:52,461 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,461 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652448) roll requested 2024-11-19T17:50:52,462 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652461 2024-11-19T17:50:52,467 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,467 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,467 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,468 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,468 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,468 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652448 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652461 2024-11-19T17:50:52,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,473 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652461) roll requested 2024-11-19T17:50:52,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652473 2024-11-19T17:50:52,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,486 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652461 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652473 2024-11-19T17:50:52,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,492 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652473) roll requested 2024-11-19T17:50:52,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652493 2024-11-19T17:50:52,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652473 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652493 2024-11-19T17:50:52,503 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,503 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652493) roll requested 2024-11-19T17:50:52,503 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652503 2024-11-19T17:50:52,508 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,508 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,508 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,508 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,508 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,508 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652493 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652503 2024-11-19T17:50:52,509 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,509 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,509 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,509 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652503) roll requested 2024-11-19T17:50:52,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652509 2024-11-19T17:50:52,532 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,532 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,532 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652503 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652509 2024-11-19T17:50:52,534 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,534 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,534 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,535 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652509) roll requested 2024-11-19T17:50:52,535 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652535 2024-11-19T17:50:52,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,540 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652509 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652535 2024-11-19T17:50:52,542 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,542 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,542 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,542 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652535) roll requested 2024-11-19T17:50:52,542 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652542 2024-11-19T17:50:52,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,551 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,552 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,552 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652535 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652542 2024-11-19T17:50:52,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,552 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652542) roll requested 2024-11-19T17:50:52,553 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652552 2024-11-19T17:50:52,558 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,558 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,558 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,558 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,558 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652542 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652552 2024-11-19T17:50:52,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,559 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,559 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,559 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652552) roll requested 2024-11-19T17:50:52,559 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652559 2024-11-19T17:50:52,567 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,567 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,567 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,567 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,567 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,567 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652552 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652559 2024-11-19T17:50:52,568 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,568 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652559) roll requested 2024-11-19T17:50:52,568 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652568 2024-11-19T17:50:52,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652559 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652568 2024-11-19T17:50:52,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,576 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652568) roll requested 2024-11-19T17:50:52,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652577 2024-11-19T17:50:52,588 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652568 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652577 2024-11-19T17:50:52,590 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,590 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,590 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,590 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652577) roll requested 2024-11-19T17:50:52,590 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652590 2024-11-19T17:50:52,603 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,603 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,603 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,603 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,603 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,603 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652577 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652590 2024-11-19T17:50:52,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,604 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652590) roll requested 2024-11-19T17:50:52,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652604 2024-11-19T17:50:52,615 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,615 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,615 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,615 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652590 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652604 2024-11-19T17:50:52,616 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,616 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,616 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,616 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652604) roll requested 2024-11-19T17:50:52,616 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652616 2024-11-19T17:50:52,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,622 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,622 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,622 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,622 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652604 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652616 2024-11-19T17:50:52,625 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,625 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652616) roll requested 2024-11-19T17:50:52,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652625 2024-11-19T17:50:52,632 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,632 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,632 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,632 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,632 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,633 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652616 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652625 2024-11-19T17:50:52,636 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,636 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,636 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,636 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652625) roll requested 2024-11-19T17:50:52,636 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652636 2024-11-19T17:50:52,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652625 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652636 2024-11-19T17:50:52,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,654 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,654 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652636) roll requested 2024-11-19T17:50:52,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652654 2024-11-19T17:50:52,672 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,674 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,674 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,674 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652636 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652654 2024-11-19T17:50:52,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,680 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,680 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652654) roll requested 2024-11-19T17:50:52,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652680 2024-11-19T17:50:52,690 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,690 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,690 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,690 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,691 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,691 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652654 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652680 2024-11-19T17:50:52,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,696 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,696 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,696 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652680) roll requested 2024-11-19T17:50:52,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652697 2024-11-19T17:50:52,746 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,746 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,746 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,746 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,746 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,746 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652680 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652697 2024-11-19T17:50:52,755 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,755 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,755 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,755 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652697) roll requested 2024-11-19T17:50:52,756 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652756 2024-11-19T17:50:52,764 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,764 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,764 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,765 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,765 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,765 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652697 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652756 2024-11-19T17:50:52,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,768 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652756) roll requested 2024-11-19T17:50:52,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652769 2024-11-19T17:50:52,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652756 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652769 2024-11-19T17:50:52,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,804 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652769) roll requested 2024-11-19T17:50:52,804 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652804 2024-11-19T17:50:52,835 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,835 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,835 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,836 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,836 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,836 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652769 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652804 2024-11-19T17:50:52,842 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,842 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,843 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,843 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652804) roll requested 2024-11-19T17:50:52,843 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652843 2024-11-19T17:50:52,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,874 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,874 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,879 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,879 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,879 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652804 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652843 2024-11-19T17:50:52,899 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,899 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,899 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,899 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652843) roll requested 2024-11-19T17:50:52,899 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652899 2024-11-19T17:50:52,915 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,916 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,916 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,916 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,916 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,916 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652843 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652899 2024-11-19T17:50:52,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,920 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652899) roll requested 2024-11-19T17:50:52,920 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652920 2024-11-19T17:50:52,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,934 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652899 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652920 2024-11-19T17:50:52,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,949 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,949 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652920) roll requested 2024-11-19T17:50:52,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652949 2024-11-19T17:50:52,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,955 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,955 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,955 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,955 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652920 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652949 2024-11-19T17:50:52,956 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,956 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,956 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,956 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652949) roll requested 2024-11-19T17:50:52,956 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652956 2024-11-19T17:50:52,964 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,964 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,964 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,964 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,965 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,965 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652949 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652956 2024-11-19T17:50:52,966 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,966 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,966 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,966 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652956) roll requested 2024-11-19T17:50:52,966 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652966 2024-11-19T17:50:52,981 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,981 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652956 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652966 2024-11-19T17:50:52,982 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,982 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,982 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,982 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652966) roll requested 2024-11-19T17:50:52,982 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652982 2024-11-19T17:50:52,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:52,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652966 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652982 2024-11-19T17:50:52,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:52,996 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,996 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:52,996 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652982) roll requested 2024-11-19T17:50:52,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038652996 2024-11-19T17:50:53,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,004 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,005 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,005 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652982 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652996 2024-11-19T17:50:53,008 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,008 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,008 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,008 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038652996) roll requested 2024-11-19T17:50:53,008 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653008 2024-11-19T17:50:53,020 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,020 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,020 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,021 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,021 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,021 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038652996 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653008 2024-11-19T17:50:53,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,021 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,022 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,022 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653008) roll requested 2024-11-19T17:50:53,022 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653022 2024-11-19T17:50:53,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,027 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,027 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,027 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,027 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,027 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653008 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653022 2024-11-19T17:50:53,028 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,028 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,028 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,028 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653022) roll requested 2024-11-19T17:50:53,028 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653028 2024-11-19T17:50:53,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,048 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,048 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,048 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,048 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653022 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653028 2024-11-19T17:50:53,051 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,051 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,051 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,051 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653028) roll requested 2024-11-19T17:50:53,052 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653052 2024-11-19T17:50:53,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,060 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,060 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,060 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653028 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653052 2024-11-19T17:50:53,061 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,061 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,062 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,062 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653052) roll requested 2024-11-19T17:50:53,062 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653062 2024-11-19T17:50:53,068 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,068 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,068 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,072 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,072 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,072 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653052 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653062 2024-11-19T17:50:53,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,073 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653062) roll requested 2024-11-19T17:50:53,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653073 2024-11-19T17:50:53,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,082 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,082 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,082 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,083 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653062 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653073 2024-11-19T17:50:53,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,083 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,083 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653073) roll requested 2024-11-19T17:50:53,083 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653083 2024-11-19T17:50:53,090 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,090 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,090 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,090 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,090 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,090 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653073 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653083 2024-11-19T17:50:53,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,096 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653083) roll requested 2024-11-19T17:50:53,096 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653096 2024-11-19T17:50:53,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653083 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653096 2024-11-19T17:50:53,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,105 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653096) roll requested 2024-11-19T17:50:53,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653105 2024-11-19T17:50:53,115 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,115 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,115 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,115 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653096 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653105 2024-11-19T17:50:53,115 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,115 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,115 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653105) roll requested 2024-11-19T17:50:53,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653116 2024-11-19T17:50:53,120 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,120 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,120 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,121 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,121 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,121 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653105 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653116 2024-11-19T17:50:53,124 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,124 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,124 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,125 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653116) roll requested 2024-11-19T17:50:53,125 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653125 2024-11-19T17:50:53,130 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,131 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,131 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,131 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,131 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,131 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653116 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653125 2024-11-19T17:50:53,132 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,132 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,132 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,132 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653125) roll requested 2024-11-19T17:50:53,132 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653132 2024-11-19T17:50:53,140 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,140 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,140 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,140 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,141 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,141 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653125 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653132 2024-11-19T17:50:53,141 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,142 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,142 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,142 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653132) roll requested 2024-11-19T17:50:53,142 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653142 2024-11-19T17:50:53,150 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,150 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,150 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,151 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653132 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653142 2024-11-19T17:50:53,151 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,151 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,151 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653142) roll requested 2024-11-19T17:50:53,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653151 2024-11-19T17:50:53,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,161 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653142 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653151 2024-11-19T17:50:53,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,162 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,162 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,162 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653151) roll requested 2024-11-19T17:50:53,162 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653162 2024-11-19T17:50:53,168 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,169 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,169 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,169 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,169 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,169 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653151 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653162 2024-11-19T17:50:53,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,172 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653162) roll requested 2024-11-19T17:50:53,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653173 2024-11-19T17:50:53,178 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,178 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,178 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653162 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653173 2024-11-19T17:50:53,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,180 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,181 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653173) roll requested 2024-11-19T17:50:53,181 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653181 2024-11-19T17:50:53,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,189 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,189 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653173 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653181 2024-11-19T17:50:53,189 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,189 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653181) roll requested 2024-11-19T17:50:53,190 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653189 2024-11-19T17:50:53,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,202 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,202 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,202 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653181 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653189 2024-11-19T17:50:53,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,208 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,209 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,209 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653189) roll requested 2024-11-19T17:50:53,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653209 2024-11-19T17:50:53,219 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,219 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,219 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,219 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653189 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653209 2024-11-19T17:50:53,220 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,220 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,220 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,220 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653209) roll requested 2024-11-19T17:50:53,220 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653220 2024-11-19T17:50:53,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,229 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,229 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,230 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653209 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653220 2024-11-19T17:50:53,231 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,231 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,231 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,231 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653220) roll requested 2024-11-19T17:50:53,231 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653231 2024-11-19T17:50:53,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,239 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,239 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,239 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,240 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653220 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653231 2024-11-19T17:50:53,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,244 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,245 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653231) roll requested 2024-11-19T17:50:53,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653245 2024-11-19T17:50:53,257 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,257 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,257 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,257 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,257 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,257 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653231 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653245 2024-11-19T17:50:53,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,264 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653245) roll requested 2024-11-19T17:50:53,265 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653264 2024-11-19T17:50:53,287 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,287 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,287 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,288 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,288 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653245 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653264 2024-11-19T17:50:53,290 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,290 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,290 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,290 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653264) roll requested 2024-11-19T17:50:53,290 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653290 2024-11-19T17:50:53,303 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,303 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,303 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,303 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,303 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653264 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653290 2024-11-19T17:50:53,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,305 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653290) roll requested 2024-11-19T17:50:53,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653305 2024-11-19T17:50:53,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,320 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,320 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,320 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,320 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653290 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653305 2024-11-19T17:50:53,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,332 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,332 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653305) roll requested 2024-11-19T17:50:53,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653332 2024-11-19T17:50:53,340 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,340 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,340 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,341 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653305 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653332 2024-11-19T17:50:53,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,348 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,348 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,348 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653332) roll requested 2024-11-19T17:50:53,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653348 2024-11-19T17:50:53,358 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,358 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,359 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,359 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,359 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,359 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653332 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653348 2024-11-19T17:50:53,362 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,362 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653348) roll requested 2024-11-19T17:50:53,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653363 2024-11-19T17:50:53,372 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,372 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,372 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,372 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,372 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,372 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653348 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653363 2024-11-19T17:50:53,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,374 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,374 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,374 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653363) roll requested 2024-11-19T17:50:53,374 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653374 2024-11-19T17:50:53,380 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,380 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,380 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,380 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,380 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,380 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653363 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653374 2024-11-19T17:50:53,381 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,381 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,381 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,382 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653374) roll requested 2024-11-19T17:50:53,383 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653382 2024-11-19T17:50:53,390 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,390 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,391 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,391 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,391 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,391 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653374 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653382 2024-11-19T17:50:53,397 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,397 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653382) roll requested 2024-11-19T17:50:53,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653397 2024-11-19T17:50:53,408 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,408 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,408 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,408 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,408 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,409 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653382 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653397 2024-11-19T17:50:53,410 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,410 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,410 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,410 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653397) roll requested 2024-11-19T17:50:53,410 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653410 2024-11-19T17:50:53,416 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,416 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,416 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,416 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,416 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,416 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653397 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653410 2024-11-19T17:50:53,417 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,417 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,417 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,417 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653410) roll requested 2024-11-19T17:50:53,417 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653417 2024-11-19T17:50:53,424 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,424 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,424 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,424 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,424 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,425 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653410 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653417 2024-11-19T17:50:53,425 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,425 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,425 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,425 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653417) roll requested 2024-11-19T17:50:53,425 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653425 2024-11-19T17:50:53,432 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,432 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,432 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,432 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653417 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653425 2024-11-19T17:50:53,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,434 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,434 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653425) roll requested 2024-11-19T17:50:53,434 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653434 2024-11-19T17:50:53,447 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,447 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,447 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,448 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,448 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653425 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653434 2024-11-19T17:50:53,448 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,448 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,448 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,448 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653434) roll requested 2024-11-19T17:50:53,449 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653448 2024-11-19T17:50:53,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,457 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,457 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,457 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,457 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653434 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653448 2024-11-19T17:50:53,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,457 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653448) roll requested 2024-11-19T17:50:53,458 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653458 2024-11-19T17:50:53,464 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653448 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653458 2024-11-19T17:50:53,466 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,466 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653458) roll requested 2024-11-19T17:50:53,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653466 2024-11-19T17:50:53,472 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,472 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,472 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,472 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653458 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653466 2024-11-19T17:50:53,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,473 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653466) roll requested 2024-11-19T17:50:53,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653473 2024-11-19T17:50:53,478 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,478 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,478 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,478 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,479 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653466 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653473 2024-11-19T17:50:53,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,480 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,480 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653473) roll requested 2024-11-19T17:50:53,480 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653480 2024-11-19T17:50:53,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,487 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,487 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653473 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653480 2024-11-19T17:50:53,487 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,487 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,487 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653480) roll requested 2024-11-19T17:50:53,488 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653487 2024-11-19T17:50:53,496 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,496 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,496 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,496 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,497 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653480 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653487 2024-11-19T17:50:53,497 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,497 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653487) roll requested 2024-11-19T17:50:53,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653497 2024-11-19T17:50:53,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,504 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,504 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,504 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653487 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653497 2024-11-19T17:50:53,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,504 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653497) roll requested 2024-11-19T17:50:53,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653505 2024-11-19T17:50:53,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,512 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,512 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,512 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,512 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653497 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653505 2024-11-19T17:50:53,513 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,513 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653505) roll requested 2024-11-19T17:50:53,514 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653514 2024-11-19T17:50:53,523 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,523 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,523 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653505 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653514 2024-11-19T17:50:53,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,524 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,524 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,524 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653514) roll requested 2024-11-19T17:50:53,524 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653524 2024-11-19T17:50:53,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,530 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,530 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653514 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653524 2024-11-19T17:50:53,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,531 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653524) roll requested 2024-11-19T17:50:53,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653531 2024-11-19T17:50:53,543 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,543 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653524 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653531 2024-11-19T17:50:53,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,544 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,544 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653531) roll requested 2024-11-19T17:50:53,544 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653544 2024-11-19T17:50:53,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,551 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,551 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,551 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653531 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653544 2024-11-19T17:50:53,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,552 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653544) roll requested 2024-11-19T17:50:53,553 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653553 2024-11-19T17:50:53,560 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,560 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,560 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,560 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,561 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,561 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653544 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653553 2024-11-19T17:50:53,561 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,561 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,561 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,561 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653553) roll requested 2024-11-19T17:50:53,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653561 2024-11-19T17:50:53,568 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,569 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,569 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,569 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,569 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653553 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653561 2024-11-19T17:50:53,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,572 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653561) roll requested 2024-11-19T17:50:53,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653572 2024-11-19T17:50:53,580 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,581 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,581 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,581 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,581 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653561 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653572 2024-11-19T17:50:53,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,584 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653572) roll requested 2024-11-19T17:50:53,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653585 2024-11-19T17:50:53,598 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,599 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,599 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,599 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,599 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,599 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653572 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653585 2024-11-19T17:50:53,599 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,599 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,599 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653585) roll requested 2024-11-19T17:50:53,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653599 2024-11-19T17:50:53,606 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,606 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,606 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,606 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653585 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653599 2024-11-19T17:50:53,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,612 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653599) roll requested 2024-11-19T17:50:53,613 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653612 2024-11-19T17:50:53,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,622 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,622 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,622 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653599 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653612 2024-11-19T17:50:53,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,624 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653612) roll requested 2024-11-19T17:50:53,624 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653624 2024-11-19T17:50:53,633 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653612 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653624 2024-11-19T17:50:53,636 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,636 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,636 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,636 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653624) roll requested 2024-11-19T17:50:53,637 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653636 2024-11-19T17:50:53,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,642 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,642 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,642 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,642 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653624 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653636 2024-11-19T17:50:53,643 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,643 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,643 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,643 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653636) roll requested 2024-11-19T17:50:53,643 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653643 2024-11-19T17:50:53,651 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,651 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,651 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653636 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653643 2024-11-19T17:50:53,652 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,652 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,652 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,652 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653643) roll requested 2024-11-19T17:50:53,653 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653653 2024-11-19T17:50:53,659 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,659 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,659 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,659 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,659 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653643 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653653 2024-11-19T17:50:53,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,660 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653653) roll requested 2024-11-19T17:50:53,661 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653661 2024-11-19T17:50:53,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,667 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,667 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,667 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653653 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653661 2024-11-19T17:50:53,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,672 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,672 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653661) roll requested 2024-11-19T17:50:53,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653672 2024-11-19T17:50:53,683 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,683 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,683 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,683 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,683 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,683 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653661 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653672 2024-11-19T17:50:53,687 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,687 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,688 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,688 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653672) roll requested 2024-11-19T17:50:53,688 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653688 2024-11-19T17:50:53,697 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,697 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,697 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,697 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,697 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,698 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653672 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653688 2024-11-19T17:50:53,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,701 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653688) roll requested 2024-11-19T17:50:53,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653701 2024-11-19T17:50:53,716 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,716 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,717 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,717 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,717 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,717 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653688 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653701 2024-11-19T17:50:53,717 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,717 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,717 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,718 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653701) roll requested 2024-11-19T17:50:53,719 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653719 2024-11-19T17:50:53,735 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,735 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,735 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,735 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,735 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653701 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653719 2024-11-19T17:50:53,737 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,737 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,737 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653719) roll requested 2024-11-19T17:50:53,737 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653737 2024-11-19T17:50:53,745 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,746 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,746 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,748 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653719 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653737 2024-11-19T17:50:53,753 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,753 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653737) roll requested 2024-11-19T17:50:53,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653753 2024-11-19T17:50:53,764 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,764 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,764 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,764 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,764 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,764 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653737 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653753 2024-11-19T17:50:53,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,768 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653753) roll requested 2024-11-19T17:50:53,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653769 2024-11-19T17:50:53,776 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,776 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,776 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,776 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,776 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,777 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653753 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653769 2024-11-19T17:50:53,780 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,780 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,780 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,780 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653769) roll requested 2024-11-19T17:50:53,781 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653780 2024-11-19T17:50:53,796 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,796 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,797 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,797 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,797 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,797 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653769 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653780 2024-11-19T17:50:53,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,801 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,801 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,801 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653780) roll requested 2024-11-19T17:50:53,801 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653801 2024-11-19T17:50:53,815 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,815 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,815 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,815 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,815 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,815 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653780 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653801 2024-11-19T17:50:53,816 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,816 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,816 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,816 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653801) roll requested 2024-11-19T17:50:53,816 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653816 2024-11-19T17:50:53,823 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,824 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,824 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653801 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653816 2024-11-19T17:50:53,825 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,825 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,825 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,825 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653816) roll requested 2024-11-19T17:50:53,825 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653825 2024-11-19T17:50:53,832 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,832 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,832 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,832 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,832 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,833 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653816 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653825 2024-11-19T17:50:53,833 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,833 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,833 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,833 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653825) roll requested 2024-11-19T17:50:53,833 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653833 2024-11-19T17:50:53,840 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653825 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653833 2024-11-19T17:50:53,841 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,841 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,841 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,841 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653833) roll requested 2024-11-19T17:50:53,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653841 2024-11-19T17:50:53,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,852 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653833 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653841 2024-11-19T17:50:53,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,856 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,856 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,856 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653841) roll requested 2024-11-19T17:50:53,857 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653856 2024-11-19T17:50:53,862 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,862 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,863 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,863 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653841 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653856 2024-11-19T17:50:53,875 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,875 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,875 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,875 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653856) roll requested 2024-11-19T17:50:53,875 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653875 2024-11-19T17:50:53,881 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,881 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,881 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,881 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,881 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,881 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653856 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653875 2024-11-19T17:50:53,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,900 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653875) roll requested 2024-11-19T17:50:53,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653900 2024-11-19T17:50:53,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,919 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,919 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653875 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653900 2024-11-19T17:50:53,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,920 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653900) roll requested 2024-11-19T17:50:53,920 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653920 2024-11-19T17:50:53,932 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,932 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,933 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653900 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653920 2024-11-19T17:50:53,944 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,944 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,944 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653920) roll requested 2024-11-19T17:50:53,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653944 2024-11-19T17:50:53,950 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,950 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,950 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,950 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,950 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,950 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653920 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653944 2024-11-19T17:50:53,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,952 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653944) roll requested 2024-11-19T17:50:53,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653952 2024-11-19T17:50:53,961 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,962 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,962 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,962 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,962 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,962 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653944 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653952 2024-11-19T17:50:53,971 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,971 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,971 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653952) roll requested 2024-11-19T17:50:53,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653972 2024-11-19T17:50:53,978 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,978 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,978 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,978 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,978 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,978 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653952 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653972 2024-11-19T17:50:53,984 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,984 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,984 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,984 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653972) roll requested 2024-11-19T17:50:53,985 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653985 2024-11-19T17:50:53,992 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,993 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,993 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,993 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,993 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:53,993 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653972 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653985 2024-11-19T17:50:53,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:53,996 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,996 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:53,996 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653985) roll requested 2024-11-19T17:50:53,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038653996 2024-11-19T17:50:54,008 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,008 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,008 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,008 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,009 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653985 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653996 2024-11-19T17:50:54,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,012 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,013 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038653996) roll requested 2024-11-19T17:50:54,013 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654013 2024-11-19T17:50:54,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038653996 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654013 2024-11-19T17:50:54,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,020 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,020 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,020 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654013) roll requested 2024-11-19T17:50:54,020 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654020 2024-11-19T17:50:54,026 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,034 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,035 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,035 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654013 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654020 2024-11-19T17:50:54,035 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,035 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,035 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,036 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654020) roll requested 2024-11-19T17:50:54,036 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654036 2024-11-19T17:50:54,051 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,051 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,052 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,052 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,052 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,052 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654020 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654036 2024-11-19T17:50:54,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,052 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654036) roll requested 2024-11-19T17:50:54,053 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654052 2024-11-19T17:50:54,066 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,066 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,066 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,066 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,066 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,067 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654036 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654052 2024-11-19T17:50:54,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,068 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,068 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,068 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654052) roll requested 2024-11-19T17:50:54,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654068 2024-11-19T17:50:54,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654052 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654068 2024-11-19T17:50:54,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,080 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,080 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654068) roll requested 2024-11-19T17:50:54,081 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654081 2024-11-19T17:50:54,090 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,090 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,091 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,091 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654068 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654081 2024-11-19T17:50:54,100 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,101 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,101 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,101 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654081) roll requested 2024-11-19T17:50:54,101 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654101 2024-11-19T17:50:54,111 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,111 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,111 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,111 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,111 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654081 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654101 2024-11-19T17:50:54,111 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,111 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,111 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,112 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654101) roll requested 2024-11-19T17:50:54,112 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654112 2024-11-19T17:50:54,132 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,133 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,133 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,133 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,133 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,133 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654101 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654112 2024-11-19T17:50:54,134 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,134 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,134 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,134 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654112) roll requested 2024-11-19T17:50:54,134 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654134 2024-11-19T17:50:54,149 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,150 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,150 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,150 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,150 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654112 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654134 2024-11-19T17:50:54,150 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,150 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,151 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654134) roll requested 2024-11-19T17:50:54,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654151 2024-11-19T17:50:54,156 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,156 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,156 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,156 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,156 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,156 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654134 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654151 2024-11-19T17:50:54,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,160 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654151) roll requested 2024-11-19T17:50:54,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654160 2024-11-19T17:50:54,166 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,166 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,166 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,166 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,166 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,166 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654151 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654160 2024-11-19T17:50:54,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,169 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,169 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,169 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654160) roll requested 2024-11-19T17:50:54,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654169 2024-11-19T17:50:54,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654160 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654169 2024-11-19T17:50:54,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,184 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,184 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654169) roll requested 2024-11-19T17:50:54,184 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654184 2024-11-19T17:50:54,197 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,197 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,197 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654169 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654184 2024-11-19T17:50:54,200 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,200 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,200 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,200 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654184) roll requested 2024-11-19T17:50:54,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654200 2024-11-19T17:50:54,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,211 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,211 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,211 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,211 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654184 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654200 2024-11-19T17:50:54,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,216 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654200) roll requested 2024-11-19T17:50:54,216 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654216 2024-11-19T17:50:54,224 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,224 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,224 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,224 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654200 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654216 2024-11-19T17:50:54,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,225 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654216) roll requested 2024-11-19T17:50:54,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654225 2024-11-19T17:50:54,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,230 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,231 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,231 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,231 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654216 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654225 2024-11-19T17:50:54,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,232 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,232 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654225) roll requested 2024-11-19T17:50:54,232 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654232 2024-11-19T17:50:54,237 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,238 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,238 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,238 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,238 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,238 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654225 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654232 2024-11-19T17:50:54,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,240 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,241 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,241 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654232) roll requested 2024-11-19T17:50:54,241 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654241 2024-11-19T17:50:54,248 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,249 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,249 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,249 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,249 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,249 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654232 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654241 2024-11-19T17:50:54,249 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,249 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,249 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,249 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654241) roll requested 2024-11-19T17:50:54,250 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654249 2024-11-19T17:50:54,256 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,256 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,256 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,256 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,257 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,257 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654241 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654249 2024-11-19T17:50:54,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,257 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654249) roll requested 2024-11-19T17:50:54,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654257 2024-11-19T17:50:54,265 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,266 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,266 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,266 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,266 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,266 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654249 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654257 2024-11-19T17:50:54,266 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,266 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,266 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,266 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654257) roll requested 2024-11-19T17:50:54,267 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654267 2024-11-19T17:50:54,272 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,272 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,273 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,273 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,273 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,273 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654257 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654267 2024-11-19T17:50:54,273 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,273 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,273 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654267) roll requested 2024-11-19T17:50:54,274 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654273 2024-11-19T17:50:54,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,279 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,279 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,279 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,279 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654267 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654273 2024-11-19T17:50:54,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,284 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654273) roll requested 2024-11-19T17:50:54,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654285 2024-11-19T17:50:54,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,291 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654273 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654285 2024-11-19T17:50:54,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,293 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654285) roll requested 2024-11-19T17:50:54,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654293 2024-11-19T17:50:54,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,301 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654285 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654293 2024-11-19T17:50:54,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,301 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654293) roll requested 2024-11-19T17:50:54,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654301 2024-11-19T17:50:54,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,312 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,312 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,312 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654293 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654301 2024-11-19T17:50:54,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,312 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654301) roll requested 2024-11-19T17:50:54,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654312 2024-11-19T17:50:54,323 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,323 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,323 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,323 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,323 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,323 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654301 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654312 2024-11-19T17:50:54,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,324 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654312) roll requested 2024-11-19T17:50:54,324 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654324 2024-11-19T17:50:54,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,339 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,339 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,339 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,339 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654312 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654324 2024-11-19T17:50:54,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,340 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,340 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654324) roll requested 2024-11-19T17:50:54,341 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654340 2024-11-19T17:50:54,348 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,348 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,348 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,349 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,349 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,349 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654324 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654340 2024-11-19T17:50:54,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,354 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,354 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,354 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654340) roll requested 2024-11-19T17:50:54,354 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654354 2024-11-19T17:50:54,364 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,364 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,364 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,364 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,364 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,365 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654340 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654354 2024-11-19T17:50:54,365 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,365 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,365 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,365 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654354) roll requested 2024-11-19T17:50:54,365 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654365 2024-11-19T17:50:54,373 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,373 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,373 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,373 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654354 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654365 2024-11-19T17:50:54,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,374 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,374 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,374 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654365) roll requested 2024-11-19T17:50:54,374 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654374 2024-11-19T17:50:54,385 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,385 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,385 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,385 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,385 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,385 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654365 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654374 2024-11-19T17:50:54,388 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,388 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,388 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,388 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654374) roll requested 2024-11-19T17:50:54,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654389 2024-11-19T17:50:54,395 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,395 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,395 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,395 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,395 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,395 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654374 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654389 2024-11-19T17:50:54,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,396 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,396 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654389) roll requested 2024-11-19T17:50:54,396 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654396 2024-11-19T17:50:54,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,403 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,403 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,403 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654389 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654396 2024-11-19T17:50:54,404 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,404 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,404 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,404 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654396) roll requested 2024-11-19T17:50:54,404 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654404 2024-11-19T17:50:54,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,412 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,412 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,412 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654396 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654404 2024-11-19T17:50:54,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,413 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,414 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,414 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654404) roll requested 2024-11-19T17:50:54,414 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654414 2024-11-19T17:50:54,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,428 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,428 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,428 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,428 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,428 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654404 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654414 2024-11-19T17:50:54,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,428 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654414) roll requested 2024-11-19T17:50:54,429 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654429 2024-11-19T17:50:54,436 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,436 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,436 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,437 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,437 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,437 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654414 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654429 2024-11-19T17:50:54,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,441 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:50:54,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654429) roll requested 2024-11-19T17:50:54,441 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654441 2024-11-19T17:50:54,457 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,457 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,457 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,457 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,457 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,457 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654429 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654441 2024-11-19T17:50:54,458 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,458 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,459 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,459 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654441) roll requested 2024-11-19T17:50:54,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654459 2024-11-19T17:50:54,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,475 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654441 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654459 2024-11-19T17:50:54,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,476 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,476 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654459) roll requested 2024-11-19T17:50:54,477 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654476 2024-11-19T17:50:54,488 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,488 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,488 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,488 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,488 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,488 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654459 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654476 2024-11-19T17:50:54,491 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,491 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,492 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,492 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654476) roll requested 2024-11-19T17:50:54,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654492 2024-11-19T17:50:54,498 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,498 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,498 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,498 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654476 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654492 2024-11-19T17:50:54,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,501 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654492) roll requested 2024-11-19T17:50:54,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654501 2024-11-19T17:50:54,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654492 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654501 2024-11-19T17:50:54,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,512 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,513 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654501) roll requested 2024-11-19T17:50:54,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654513 2024-11-19T17:50:54,524 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,525 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,525 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,525 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,525 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,525 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654501 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654513 2024-11-19T17:50:54,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,528 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,529 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654513) roll requested 2024-11-19T17:50:54,529 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654529 2024-11-19T17:50:54,535 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,535 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,535 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,535 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,535 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,535 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654513 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654529 2024-11-19T17:50:54,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,537 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654529) roll requested 2024-11-19T17:50:54,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654537 2024-11-19T17:50:54,544 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,544 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,544 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,544 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,545 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,545 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654529 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654537 2024-11-19T17:50:54,547 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,547 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,547 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,547 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654537) roll requested 2024-11-19T17:50:54,547 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654547 2024-11-19T17:50:54,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654537 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654547 2024-11-19T17:50:54,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,572 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654547) roll requested 2024-11-19T17:50:54,573 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654572 2024-11-19T17:50:54,580 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,580 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,580 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,581 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,581 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654547 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654572 2024-11-19T17:50:54,581 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,581 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,582 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,582 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654572) roll requested 2024-11-19T17:50:54,582 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654582 2024-11-19T17:50:54,599 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,599 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,599 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,599 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,606 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654572 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654582 2024-11-19T17:50:54,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,620 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,621 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654582) roll requested 2024-11-19T17:50:54,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654621 2024-11-19T17:50:54,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,642 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,642 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,642 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,642 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654582 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654621 2024-11-19T17:50:54,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,644 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,644 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654621) roll requested 2024-11-19T17:50:54,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654645 2024-11-19T17:50:54,650 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,650 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,650 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,650 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,650 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654621 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654645 2024-11-19T17:50:54,655 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,656 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654645) roll requested 2024-11-19T17:50:54,656 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654656 2024-11-19T17:50:54,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,662 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,662 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,662 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,662 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,662 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654645 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654656 2024-11-19T17:50:54,665 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,665 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,665 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,665 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654656) roll requested 2024-11-19T17:50:54,665 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654665 2024-11-19T17:50:54,672 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,672 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,672 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,672 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,673 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654656 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654665 2024-11-19T17:50:54,681 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,681 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,681 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,682 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654665) roll requested 2024-11-19T17:50:54,682 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654682 2024-11-19T17:50:54,691 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,691 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,691 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,691 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,691 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,691 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654665 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654682 2024-11-19T17:50:54,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,701 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654682) roll requested 2024-11-19T17:50:54,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654701 2024-11-19T17:50:54,706 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654682 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654701 2024-11-19T17:50:54,709 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,709 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654701) roll requested 2024-11-19T17:50:54,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654709 2024-11-19T17:50:54,715 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654701 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654709 2024-11-19T17:50:54,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,716 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,716 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,717 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654709) roll requested 2024-11-19T17:50:54,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654717 2024-11-19T17:50:54,723 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,723 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,723 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,723 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,723 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,723 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654709 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654717 2024-11-19T17:50:54,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,724 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,724 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654717) roll requested 2024-11-19T17:50:54,724 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654724 2024-11-19T17:50:54,742 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,743 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,743 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,743 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,743 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,743 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654717 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654724 2024-11-19T17:50:54,745 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,745 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,745 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654724) roll requested 2024-11-19T17:50:54,746 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654746 2024-11-19T17:50:54,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,752 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654724 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654746 2024-11-19T17:50:54,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,752 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654746) roll requested 2024-11-19T17:50:54,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654753 2024-11-19T17:50:54,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654746 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654753 2024-11-19T17:50:54,769 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,769 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,769 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654753) roll requested 2024-11-19T17:50:54,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654769 2024-11-19T17:50:54,775 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,775 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654753 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654769 2024-11-19T17:50:54,783 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,784 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654769) roll requested 2024-11-19T17:50:54,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654784 2024-11-19T17:50:54,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,795 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,795 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654769 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654784 2024-11-19T17:50:54,795 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,796 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654784) roll requested 2024-11-19T17:50:54,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654796 2024-11-19T17:50:54,809 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,809 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,809 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,809 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,809 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,810 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654784 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654796 2024-11-19T17:50:54,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,821 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654796) roll requested 2024-11-19T17:50:54,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654821 2024-11-19T17:50:54,834 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,834 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,834 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,834 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,834 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654796 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654821 2024-11-19T17:50:54,835 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,836 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,836 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654821) roll requested 2024-11-19T17:50:54,836 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654836 2024-11-19T17:50:54,851 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,851 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,852 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,852 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,852 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,852 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654821 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654836 2024-11-19T17:50:54,853 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,853 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,853 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,853 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654836) roll requested 2024-11-19T17:50:54,853 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654853 2024-11-19T17:50:54,868 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,868 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,868 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,868 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,869 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,869 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654836 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654853 2024-11-19T17:50:54,869 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,870 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,870 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,870 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654853) roll requested 2024-11-19T17:50:54,870 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654870 2024-11-19T17:50:54,876 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,876 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,876 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,876 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,876 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,876 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654853 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654870 2024-11-19T17:50:54,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,880 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654870) roll requested 2024-11-19T17:50:54,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654881 2024-11-19T17:50:54,887 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,887 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,887 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,887 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,887 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654870 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654881 2024-11-19T17:50:54,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,894 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,894 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654881) roll requested 2024-11-19T17:50:54,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654894 2024-11-19T17:50:54,902 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,903 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,903 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,903 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,903 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,903 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654881 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654894 2024-11-19T17:50:54,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,909 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,909 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654894) roll requested 2024-11-19T17:50:54,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654909 2024-11-19T17:50:54,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,926 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,926 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,926 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654894 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654909 2024-11-19T17:50:54,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,937 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,937 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654909) roll requested 2024-11-19T17:50:54,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654937 2024-11-19T17:50:54,960 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,960 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,960 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,961 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,961 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,961 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654909 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654937 2024-11-19T17:50:54,966 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,966 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,966 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,966 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654937) roll requested 2024-11-19T17:50:54,966 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654966 2024-11-19T17:50:54,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,979 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,980 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,980 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,980 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654937 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654966 2024-11-19T17:50:54,981 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:54,981 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654937 is not closed yet, will try archiving it next time 2024-11-19T17:50:54,981 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,981 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:54,981 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654966) roll requested 2024-11-19T17:50:54,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038654981 2024-11-19T17:50:54,988 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,988 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,989 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,989 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:54,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654966 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654981 2024-11-19T17:50:55,000 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,000 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038654981) roll requested 2024-11-19T17:50:55,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655000 2024-11-19T17:50:55,020 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,021 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,021 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,021 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,021 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,021 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038654981 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655000 2024-11-19T17:50:55,022 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,023 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,023 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,023 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655000) roll requested 2024-11-19T17:50:55,023 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655023 2024-11-19T17:50:55,039 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,040 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655000 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655023 2024-11-19T17:50:55,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,044 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,045 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655023) roll requested 2024-11-19T17:50:55,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655045 2024-11-19T17:50:55,052 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,052 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,052 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,053 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,053 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,053 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655023 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655045 2024-11-19T17:50:55,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,068 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,069 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655045) roll requested 2024-11-19T17:50:55,069 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655069 2024-11-19T17:50:55,083 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,083 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,084 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655045 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655069 2024-11-19T17:50:55,084 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,085 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,085 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655069) roll requested 2024-11-19T17:50:55,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655085 2024-11-19T17:50:55,109 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,110 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,110 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,110 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,110 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,110 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655069 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655085 2024-11-19T17:50:55,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,112 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,112 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,112 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655085) roll requested 2024-11-19T17:50:55,112 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655112 2024-11-19T17:50:55,149 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,149 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,149 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,149 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,149 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,149 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655085 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655112 2024-11-19T17:50:55,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,185 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,185 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655112) roll requested 2024-11-19T17:50:55,186 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655185 2024-11-19T17:50:55,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655112 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655185 2024-11-19T17:50:55,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,225 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655185) roll requested 2024-11-19T17:50:55,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655225 2024-11-19T17:50:55,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655185 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655225 2024-11-19T17:50:55,262 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,262 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,262 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,262 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655225) roll requested 2024-11-19T17:50:55,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655263 2024-11-19T17:50:55,268 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,269 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,270 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,270 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,270 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655225 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655263 2024-11-19T17:50:55,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,272 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,273 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655263) roll requested 2024-11-19T17:50:55,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655273 2024-11-19T17:50:55,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,286 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,286 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655263 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655273 2024-11-19T17:50:55,291 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,292 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655273) roll requested 2024-11-19T17:50:55,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655292 2024-11-19T17:50:55,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,301 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,301 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655273 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655292 2024-11-19T17:50:55,302 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,302 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,302 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,302 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655292) roll requested 2024-11-19T17:50:55,302 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655302 2024-11-19T17:50:55,310 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,310 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,310 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655292 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655302 2024-11-19T17:50:55,311 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,311 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,311 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,311 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655302) roll requested 2024-11-19T17:50:55,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655312 2024-11-19T17:50:55,320 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,320 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,320 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,320 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,320 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655302 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655312 2024-11-19T17:50:55,321 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,321 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,321 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,321 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655312) roll requested 2024-11-19T17:50:55,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655321 2024-11-19T17:50:55,330 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,330 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,330 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655312 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655321 2024-11-19T17:50:55,331 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,331 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,331 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,331 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655321) roll requested 2024-11-19T17:50:55,332 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655332 2024-11-19T17:50:55,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,338 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655321 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655332 2024-11-19T17:50:55,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,341 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,341 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,341 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655332) roll requested 2024-11-19T17:50:55,341 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655341 2024-11-19T17:50:55,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,346 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,346 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,347 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,347 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,347 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655332 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655341 2024-11-19T17:50:55,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,348 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655341) roll requested 2024-11-19T17:50:55,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655349 2024-11-19T17:50:55,354 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,354 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,354 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,354 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,354 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655341 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655349 2024-11-19T17:50:55,357 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,357 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,357 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,357 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655349) roll requested 2024-11-19T17:50:55,357 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655357 2024-11-19T17:50:55,362 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,362 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,363 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655349 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655357 2024-11-19T17:50:55,364 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,364 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,364 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,364 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655357) roll requested 2024-11-19T17:50:55,365 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655365 2024-11-19T17:50:55,369 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,370 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,370 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,370 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655357 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655365 2024-11-19T17:50:55,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,372 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,373 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,373 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655365) roll requested 2024-11-19T17:50:55,373 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655373 2024-11-19T17:50:55,380 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,380 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,380 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,380 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,380 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,381 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655365 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655373 2024-11-19T17:50:55,385 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,385 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,385 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,385 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655373) roll requested 2024-11-19T17:50:55,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655385 2024-11-19T17:50:55,390 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,391 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,391 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,391 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,391 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,391 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655373 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655385 2024-11-19T17:50:55,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,392 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,392 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,393 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=56, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:55,394 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:55,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655385) roll requested 2024-11-19T17:50:55,394 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655394 2024-11-19T17:50:55,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,400 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655385 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655394 2024-11-19T17:50:55,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,400 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=57, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:55,401 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,401 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,401 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:55,401 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655394) roll requested 2024-11-19T17:50:55,401 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655401 2024-11-19T17:50:55,408 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,408 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,409 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,409 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,409 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,409 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655394 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655401 2024-11-19T17:50:55,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,440 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,440 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=58, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:55,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,440 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:55,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655401) roll requested 2024-11-19T17:50:55,441 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655440 2024-11-19T17:50:55,447 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,447 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,447 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,451 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,451 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655401 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655440 2024-11-19T17:50:55,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,452 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=59, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:55,452 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,452 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:55,452 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655440) roll requested 2024-11-19T17:50:55,452 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655452 2024-11-19T17:50:55,458 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,458 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,458 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,459 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,459 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,459 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655440 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655452 2024-11-19T17:50:55,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,460 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=60, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:55,460 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,461 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:55,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655452) roll requested 2024-11-19T17:50:55,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655461 2024-11-19T17:50:55,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,472 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655452 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655461 2024-11-19T17:50:55,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,473 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=61, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:55,473 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,473 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:55,474 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655461) roll requested 2024-11-19T17:50:55,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655474 2024-11-19T17:50:55,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,479 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,479 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655461 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655474 2024-11-19T17:50:55,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,480 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=62, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:55,480 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,480 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:55,480 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655474) roll requested 2024-11-19T17:50:55,480 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655480 2024-11-19T17:50:55,494 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,494 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,494 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,494 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,494 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,494 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655474 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655480 2024-11-19T17:50:55,495 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,495 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=63, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:55,495 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,495 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:55,495 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,495 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655480) roll requested 2024-11-19T17:50:55,495 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655495 2024-11-19T17:50:55,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,520 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,520 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655480 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655495 2024-11-19T17:50:55,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,521 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=64, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:55,521 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:55,522 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655495) roll requested 2024-11-19T17:50:55,523 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655523 2024-11-19T17:50:55,535 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,535 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,535 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,535 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,535 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,535 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655495 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655523 2024-11-19T17:50:55,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,536 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=65, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:55,536 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:55,536 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,536 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,536 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655523) roll requested 2024-11-19T17:50:55,536 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655536 2024-11-19T17:50:55,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,549 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,549 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655523 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655536 2024-11-19T17:50:55,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,552 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=66, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:55,552 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,553 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,553 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:55,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655536) roll requested 2024-11-19T17:50:55,553 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655553 2024-11-19T17:50:55,560 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,560 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,560 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,560 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,560 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,561 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655536 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655553 2024-11-19T17:50:55,561 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,562 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=67, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:55,562 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,562 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,562 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:55,562 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655553) roll requested 2024-11-19T17:50:55,562 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655562 2024-11-19T17:50:55,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655553 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655562 2024-11-19T17:50:55,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,571 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=68, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:55,571 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,571 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:55,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655562) roll requested 2024-11-19T17:50:55,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655571 2024-11-19T17:50:55,585 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,585 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,585 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,585 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,585 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,585 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655562 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655571 2024-11-19T17:50:55,586 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,586 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=69, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:55,586 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:55,586 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,586 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,586 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655571) roll requested 2024-11-19T17:50:55,586 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655586 2024-11-19T17:50:55,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,593 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,593 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,593 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655571 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655586 2024-11-19T17:50:55,598 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,598 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=70, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:55,598 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,598 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,598 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:55,598 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655586) roll requested 2024-11-19T17:50:55,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655598 2024-11-19T17:50:55,605 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,606 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,606 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,606 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655586 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655598 2024-11-19T17:50:55,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,608 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=71, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:55,608 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:55,614 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,614 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,614 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655598) roll requested 2024-11-19T17:50:55,615 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655614 2024-11-19T17:50:55,634 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655598 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655614 2024-11-19T17:50:55,645 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,646 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=72, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:55,646 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,646 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,646 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:55,646 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655614) roll requested 2024-11-19T17:50:55,646 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655646 2024-11-19T17:50:55,651 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,651 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,651 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655614 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655646 2024-11-19T17:50:55,653 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,653 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=73, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:55,653 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,653 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:55,653 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,653 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655646) roll requested 2024-11-19T17:50:55,653 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655653 2024-11-19T17:50:55,662 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,662 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,662 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,662 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,662 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,662 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655646 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655653 2024-11-19T17:50:55,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,664 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=74, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:55,664 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,664 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:55,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655653) roll requested 2024-11-19T17:50:55,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655664 2024-11-19T17:50:55,670 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,670 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,670 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,670 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,671 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,671 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655653 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655664 2024-11-19T17:50:55,671 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,671 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=75, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:55,671 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,671 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:55,671 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,672 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655664) roll requested 2024-11-19T17:50:55,672 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655672 2024-11-19T17:50:55,677 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,677 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,677 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,678 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655664 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655672 2024-11-19T17:50:55,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,680 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=76, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:55,680 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,680 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:55,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655672) roll requested 2024-11-19T17:50:55,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655681 2024-11-19T17:50:55,686 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,686 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,686 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,686 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,686 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655672 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655681 2024-11-19T17:50:55,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,688 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=77, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:55,688 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,688 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,688 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:55,688 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655681) roll requested 2024-11-19T17:50:55,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655689 2024-11-19T17:50:55,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,695 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,695 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655681 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655689 2024-11-19T17:50:55,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,696 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=78, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:55,696 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:55,696 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655689) roll requested 2024-11-19T17:50:55,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655697 2024-11-19T17:50:55,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655689 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655697 2024-11-19T17:50:55,703 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,704 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=79, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:55,704 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,704 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:55,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655697) roll requested 2024-11-19T17:50:55,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655704 2024-11-19T17:50:55,709 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,709 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,709 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,709 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,709 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,710 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655697 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655704 2024-11-19T17:50:55,712 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,712 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=80, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:55,712 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,713 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:55,713 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,713 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655704) roll requested 2024-11-19T17:50:55,713 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655713 2024-11-19T17:50:55,720 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,720 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,720 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,720 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,720 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,720 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655704 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655713 2024-11-19T17:50:55,721 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,721 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=81, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:55,721 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:55,721 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,721 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655713) roll requested 2024-11-19T17:50:55,722 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655721 2024-11-19T17:50:55,727 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,727 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,728 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,728 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,728 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,728 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655713 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655721 2024-11-19T17:50:55,728 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,729 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=82, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:55,729 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:55,729 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,729 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655721) roll requested 2024-11-19T17:50:55,729 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655729 2024-11-19T17:50:55,734 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,734 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655721 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655729 2024-11-19T17:50:55,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,736 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=83, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:55,736 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,736 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:55,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,736 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655729) roll requested 2024-11-19T17:50:55,737 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655737 2024-11-19T17:50:55,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,742 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,742 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,742 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,742 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655729 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655737 2024-11-19T17:50:55,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,744 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=84, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:55,744 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:55,744 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,745 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655737) roll requested 2024-11-19T17:50:55,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655745 2024-11-19T17:50:55,752 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,752 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,752 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,753 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655737 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655745 2024-11-19T17:50:55,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,756 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=85, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:55,756 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:55,756 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655745) roll requested 2024-11-19T17:50:55,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655757 2024-11-19T17:50:55,764 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,764 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,764 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,764 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,764 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,765 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655745 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655757 2024-11-19T17:50:55,765 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,765 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=86, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:55,765 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:55,767 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655757) roll requested 2024-11-19T17:50:55,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655767 2024-11-19T17:50:55,773 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,773 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655757 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655767 2024-11-19T17:50:55,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,776 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=87, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:55,776 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,776 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:55,777 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655767) roll requested 2024-11-19T17:50:55,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655777 2024-11-19T17:50:55,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,782 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,782 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,782 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,782 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,782 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655767 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655777 2024-11-19T17:50:55,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,784 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=88, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:55,784 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,785 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,785 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:55,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655777) roll requested 2024-11-19T17:50:55,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655785 2024-11-19T17:50:55,789 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,789 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,790 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,790 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655777 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655785 2024-11-19T17:50:55,792 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,792 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=89, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:55,792 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,792 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,792 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:55,792 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655785) roll requested 2024-11-19T17:50:55,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655793 2024-11-19T17:50:55,799 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,799 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,799 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655785 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655793 2024-11-19T17:50:55,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,800 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=90, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:55,800 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,800 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:55,800 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,800 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655793) roll requested 2024-11-19T17:50:55,801 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655800 2024-11-19T17:50:55,806 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,806 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,806 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,806 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,806 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,806 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655793 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655800 2024-11-19T17:50:55,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,808 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=91, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:55,808 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,809 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,809 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:55,809 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655800) roll requested 2024-11-19T17:50:55,809 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655809 2024-11-19T17:50:55,818 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,818 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,818 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,818 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,818 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,818 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655800 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655809 2024-11-19T17:50:55,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,821 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=92, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:55,821 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,821 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:55,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655809) roll requested 2024-11-19T17:50:55,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655821 2024-11-19T17:50:55,826 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,826 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,826 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,826 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655809 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655821 2024-11-19T17:50:55,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,828 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=93, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:55,828 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,828 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:55,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655821) roll requested 2024-11-19T17:50:55,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655829 2024-11-19T17:50:55,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,837 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,837 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,837 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655821 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655829 2024-11-19T17:50:55,838 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,838 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=94, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:55,838 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,838 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:55,838 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,838 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655829) roll requested 2024-11-19T17:50:55,838 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655838 2024-11-19T17:50:55,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655829 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655838 2024-11-19T17:50:55,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,848 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=95, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:55,848 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,848 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,848 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:55,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655838) roll requested 2024-11-19T17:50:55,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655848 2024-11-19T17:50:55,853 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,854 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,854 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655838 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655848 2024-11-19T17:50:55,858 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,859 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=96, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:55,859 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:55,859 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,859 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,859 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655848) roll requested 2024-11-19T17:50:55,859 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655859 2024-11-19T17:50:55,864 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,864 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,864 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,865 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,865 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655848 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655859 2024-11-19T17:50:55,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,865 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=97, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:55,865 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,865 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:55,865 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655859) roll requested 2024-11-19T17:50:55,866 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655865 2024-11-19T17:50:55,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,873 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,873 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655859 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655865 2024-11-19T17:50:55,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,873 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=98, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:55,873 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,874 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:55,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655865) roll requested 2024-11-19T17:50:55,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655874 2024-11-19T17:50:55,882 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,882 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,882 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,882 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,882 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,882 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655865 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655874 2024-11-19T17:50:55,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,883 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=99, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:55,883 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,883 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:55,883 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,883 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655874) roll requested 2024-11-19T17:50:55,883 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655883 2024-11-19T17:50:55,888 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,888 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,888 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,888 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,888 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,888 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655874 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655883 2024-11-19T17:50:55,889 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,889 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=100, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:55,889 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:55,889 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,889 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,889 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655883) roll requested 2024-11-19T17:50:55,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655889 2024-11-19T17:50:55,898 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,898 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,898 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,898 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,898 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,898 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655883 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655889 2024-11-19T17:50:55,899 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,899 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=101, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:55,899 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,899 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,899 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:55,899 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655889) roll requested 2024-11-19T17:50:55,899 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655899 2024-11-19T17:50:55,904 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,904 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,904 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,904 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,904 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,904 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655889 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655899 2024-11-19T17:50:55,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,908 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=102, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:55,908 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,908 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:55,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655899) roll requested 2024-11-19T17:50:55,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655909 2024-11-19T17:50:55,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,913 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,914 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,914 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655899 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655909 2024-11-19T17:50:55,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,914 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=103, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:55,914 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,914 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:55,914 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,914 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655909) roll requested 2024-11-19T17:50:55,914 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655914 2024-11-19T17:50:55,919 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655909 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655914 2024-11-19T17:50:55,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,920 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=104, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:55,920 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,920 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:55,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655914) roll requested 2024-11-19T17:50:55,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655921 2024-11-19T17:50:55,925 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,925 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,925 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655914 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655921 2024-11-19T17:50:55,926 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,927 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=105, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:55,927 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,927 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,927 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:55,927 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655921) roll requested 2024-11-19T17:50:55,927 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655927 2024-11-19T17:50:55,932 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,932 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,944 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655921 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655927 2024-11-19T17:50:55,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,945 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=106, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:55,945 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,945 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:55,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655927) roll requested 2024-11-19T17:50:55,946 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655946 2024-11-19T17:50:55,955 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,955 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,955 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,956 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,956 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:55,956 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655927 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655946 2024-11-19T17:50:55,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:55,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655927 is not closed yet, will try archiving it next time 2024-11-19T17:50:55,964 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=107, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:55,964 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:55,965 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:55,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655946) roll requested 2024-11-19T17:50:55,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038655965 2024-11-19T17:50:56,008 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,008 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,008 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,008 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,008 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,009 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655946 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655965 2024-11-19T17:50:56,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,012 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=108, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,012 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,013 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,013 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,013 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038655965) roll requested 2024-11-19T17:50:56,013 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656013 2024-11-19T17:50:56,020 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,020 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,020 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,020 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,021 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,021 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038655965 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656013 2024-11-19T17:50:56,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,022 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=109, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,022 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,022 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,022 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,022 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656013) roll requested 2024-11-19T17:50:56,022 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656022 2024-11-19T17:50:56,035 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,036 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,036 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,036 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,036 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656013 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656022 2024-11-19T17:50:56,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,037 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=110, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,037 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,037 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656022) roll requested 2024-11-19T17:50:56,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656037 2024-11-19T17:50:56,044 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,045 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,045 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,045 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,045 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,045 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656022 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656037 2024-11-19T17:50:56,045 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,046 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=111, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,046 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,046 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,046 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,046 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656037) roll requested 2024-11-19T17:50:56,046 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656046 2024-11-19T17:50:56,052 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,053 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,053 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,053 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,053 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,053 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656037 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656046 2024-11-19T17:50:56,053 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,054 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=112, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,054 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,054 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,054 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,054 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656046) roll requested 2024-11-19T17:50:56,054 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656054 2024-11-19T17:50:56,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,063 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,063 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656046 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656054 2024-11-19T17:50:56,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,068 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=113, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,068 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,068 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,068 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,068 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656054) roll requested 2024-11-19T17:50:56,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656068 2024-11-19T17:50:56,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656054 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656068 2024-11-19T17:50:56,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,080 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=114, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,080 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,081 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,081 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,081 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656068) roll requested 2024-11-19T17:50:56,081 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656081 2024-11-19T17:50:56,086 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,086 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,086 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,086 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,086 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,086 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656068 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656081 2024-11-19T17:50:56,088 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,088 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=115, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,088 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,089 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656081) roll requested 2024-11-19T17:50:56,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656089 2024-11-19T17:50:56,098 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,099 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,099 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,099 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,099 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,099 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656081 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656089 2024-11-19T17:50:56,104 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,104 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=116, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,104 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,105 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656089) roll requested 2024-11-19T17:50:56,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656105 2024-11-19T17:50:56,110 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,110 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,111 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,111 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,111 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,111 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656089 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656105 2024-11-19T17:50:56,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,112 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=117, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,113 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,113 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,113 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656105) roll requested 2024-11-19T17:50:56,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656113 2024-11-19T17:50:56,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,118 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656105 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656113 2024-11-19T17:50:56,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,120 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=118, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,121 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,125 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,125 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,126 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656113) roll requested 2024-11-19T17:50:56,126 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656126 2024-11-19T17:50:56,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,136 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,137 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,137 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,137 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656113 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656126 2024-11-19T17:50:56,137 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,138 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=119, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,138 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,138 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,138 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,138 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656126) roll requested 2024-11-19T17:50:56,138 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656138 2024-11-19T17:50:56,144 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,144 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,144 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,144 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,144 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,144 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656126 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656138 2024-11-19T17:50:56,144 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,145 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=120, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,145 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,145 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656138) roll requested 2024-11-19T17:50:56,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656145 2024-11-19T17:50:56,150 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,150 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,150 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,150 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,150 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656138 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656145 2024-11-19T17:50:56,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,152 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=121, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,152 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,152 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656145) roll requested 2024-11-19T17:50:56,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656153 2024-11-19T17:50:56,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656145 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656153 2024-11-19T17:50:56,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,160 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=122, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,160 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,161 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656153) roll requested 2024-11-19T17:50:56,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656161 2024-11-19T17:50:56,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,170 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,170 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,170 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,170 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656153 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656161 2024-11-19T17:50:56,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,173 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=123, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,173 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,173 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,173 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656161) roll requested 2024-11-19T17:50:56,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656173 2024-11-19T17:50:56,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656161 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656173 2024-11-19T17:50:56,183 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,184 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=124, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,184 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,184 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,184 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656173) roll requested 2024-11-19T17:50:56,184 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656184 2024-11-19T17:50:56,192 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,192 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,192 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,192 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,192 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,193 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656173 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656184 2024-11-19T17:50:56,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,197 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=125, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,197 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,200 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,200 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,200 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656184) roll requested 2024-11-19T17:50:56,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656200 2024-11-19T17:50:56,206 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,206 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,206 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,206 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,206 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,206 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656184 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656200 2024-11-19T17:50:56,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,208 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=126, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,208 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,209 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,209 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,209 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656200) roll requested 2024-11-19T17:50:56,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656209 2024-11-19T17:50:56,214 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,214 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656200 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656209 2024-11-19T17:50:56,215 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,215 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=127, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,215 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,216 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656209) roll requested 2024-11-19T17:50:56,216 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656216 2024-11-19T17:50:56,221 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,221 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,221 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,221 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656209 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656216 2024-11-19T17:50:56,222 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,222 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=128, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,222 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,222 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,222 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,222 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656216) roll requested 2024-11-19T17:50:56,223 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656223 2024-11-19T17:50:56,235 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,235 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,235 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,235 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,235 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,235 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656216 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656223 2024-11-19T17:50:56,236 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,236 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=129, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,236 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,236 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,236 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,236 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656223) roll requested 2024-11-19T17:50:56,236 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656236 2024-11-19T17:50:56,241 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,241 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,242 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,242 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,242 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,242 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656223 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656236 2024-11-19T17:50:56,243 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,244 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=130, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,244 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,244 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,244 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656236) roll requested 2024-11-19T17:50:56,244 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656244 2024-11-19T17:50:56,249 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,249 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,249 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656236 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656244 2024-11-19T17:50:56,250 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,250 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=131, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,250 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,250 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,250 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,251 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656244) roll requested 2024-11-19T17:50:56,251 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656251 2024-11-19T17:50:56,256 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,256 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,256 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,257 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,257 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,257 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656244 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656251 2024-11-19T17:50:56,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,258 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=132, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,258 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,258 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,258 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,258 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656251) roll requested 2024-11-19T17:50:56,258 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656258 2024-11-19T17:50:56,264 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,264 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,264 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,264 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,264 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656251 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656258 2024-11-19T17:50:56,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,268 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=133, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,269 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,269 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,269 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656258) roll requested 2024-11-19T17:50:56,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656269 2024-11-19T17:50:56,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,275 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,275 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,275 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,275 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,275 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656258 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656269 2024-11-19T17:50:56,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,284 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=134, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,284 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,285 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656269) roll requested 2024-11-19T17:50:56,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656285 2024-11-19T17:50:56,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,290 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,290 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,291 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656269 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656285 2024-11-19T17:50:56,291 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,292 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=135, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,292 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,292 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656285) roll requested 2024-11-19T17:50:56,292 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656292 2024-11-19T17:50:56,297 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,297 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,297 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,297 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,297 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,298 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656285 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656292 2024-11-19T17:50:56,298 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,298 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=136, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,298 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,298 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,299 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,299 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656292) roll requested 2024-11-19T17:50:56,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656299 2024-11-19T17:50:56,304 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,304 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,304 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,304 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656292 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656299 2024-11-19T17:50:56,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,316 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=137, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,316 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,316 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,316 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656299) roll requested 2024-11-19T17:50:56,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656317 2024-11-19T17:50:56,324 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,324 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,324 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,325 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,325 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656299 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656317 2024-11-19T17:50:56,325 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,326 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=138, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,326 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,326 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,326 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,326 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656317) roll requested 2024-11-19T17:50:56,326 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656326 2024-11-19T17:50:56,331 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,331 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,331 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,331 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,331 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,331 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656317 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656326 2024-11-19T17:50:56,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,332 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=139, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,332 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,332 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,332 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656326) roll requested 2024-11-19T17:50:56,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656332 2024-11-19T17:50:56,337 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,337 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,337 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,337 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656326 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656332 2024-11-19T17:50:56,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,340 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=140, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,340 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,340 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,341 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656332) roll requested 2024-11-19T17:50:56,341 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656341 2024-11-19T17:50:56,345 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,345 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,345 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656332 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656341 2024-11-19T17:50:56,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,348 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=141, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,348 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,349 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656341) roll requested 2024-11-19T17:50:56,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656349 2024-11-19T17:50:56,356 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,356 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,356 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,356 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,357 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,357 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656341 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656349 2024-11-19T17:50:56,357 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,358 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=142, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,357 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,358 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,358 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,358 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656349) roll requested 2024-11-19T17:50:56,358 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656358 2024-11-19T17:50:56,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656349 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656358 2024-11-19T17:50:56,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,368 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=143, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,368 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,368 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656358) roll requested 2024-11-19T17:50:56,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656368 2024-11-19T17:50:56,379 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,379 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,380 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,380 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,380 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,380 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656358 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656368 2024-11-19T17:50:56,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,380 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=144, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,380 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,380 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,381 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656368) roll requested 2024-11-19T17:50:56,381 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656381 2024-11-19T17:50:56,385 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,386 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,386 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,386 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656368 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656381 2024-11-19T17:50:56,388 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,388 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=145, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,388 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,389 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656381) roll requested 2024-11-19T17:50:56,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656389 2024-11-19T17:50:56,397 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,397 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,397 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,397 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,397 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,398 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656381 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656389 2024-11-19T17:50:56,401 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,401 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=146, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,401 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,401 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,401 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,401 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656389) roll requested 2024-11-19T17:50:56,402 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656401 2024-11-19T17:50:56,410 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,411 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,411 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,411 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,411 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,411 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656389 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656401 2024-11-19T17:50:56,411 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,411 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=147, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,412 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,412 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656401) roll requested 2024-11-19T17:50:56,412 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656412 2024-11-19T17:50:56,418 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,418 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,418 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,418 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,418 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,418 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656401 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656412 2024-11-19T17:50:56,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,420 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=148, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,420 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,420 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656412) roll requested 2024-11-19T17:50:56,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656421 2024-11-19T17:50:56,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656412 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656421 2024-11-19T17:50:56,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,427 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=149, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,427 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,427 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656421) roll requested 2024-11-19T17:50:56,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656428 2024-11-19T17:50:56,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,441 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656421 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656428 2024-11-19T17:50:56,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,441 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=150, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,441 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,441 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656428) roll requested 2024-11-19T17:50:56,442 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656442 2024-11-19T17:50:56,447 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,448 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,448 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,448 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,448 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,448 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656428 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656442 2024-11-19T17:50:56,450 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,450 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=151, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,450 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,450 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656442) roll requested 2024-11-19T17:50:56,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656450 2024-11-19T17:50:56,457 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,458 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,459 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,459 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,459 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656442 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656450 2024-11-19T17:50:56,461 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,461 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=152, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,461 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,461 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656450) roll requested 2024-11-19T17:50:56,462 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656462 2024-11-19T17:50:56,467 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,467 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,467 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,467 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,467 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,467 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656450 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656462 2024-11-19T17:50:56,468 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,468 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=153, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,468 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,468 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,469 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,469 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656462) roll requested 2024-11-19T17:50:56,469 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656469 2024-11-19T17:50:56,481 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,481 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,482 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,482 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656462 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656469 2024-11-19T17:50:56,482 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,482 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=154, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,483 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,483 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656469) roll requested 2024-11-19T17:50:56,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656483 2024-11-19T17:50:56,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,502 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,502 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,502 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656469 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656483 2024-11-19T17:50:56,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,504 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=155, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,504 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,504 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656483) roll requested 2024-11-19T17:50:56,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656504 2024-11-19T17:50:56,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,509 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,509 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,509 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,510 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,510 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656483 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656504 2024-11-19T17:50:56,510 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,510 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=156, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,510 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,510 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,510 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,510 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656504) roll requested 2024-11-19T17:50:56,510 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656510 2024-11-19T17:50:56,518 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,519 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,519 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,519 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,519 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,519 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656504 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656510 2024-11-19T17:50:56,519 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,519 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=157, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,519 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,519 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,519 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,520 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656510) roll requested 2024-11-19T17:50:56,520 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656520 2024-11-19T17:50:56,527 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,528 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,528 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656510 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656520 2024-11-19T17:50:56,532 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,532 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=158, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,532 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,532 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,532 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656520) roll requested 2024-11-19T17:50:56,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656533 2024-11-19T17:50:56,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,539 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,539 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656520 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656533 2024-11-19T17:50:56,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,539 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=159, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,540 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,540 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,540 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,540 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656533) roll requested 2024-11-19T17:50:56,540 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656540 2024-11-19T17:50:56,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,549 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656533 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656540 2024-11-19T17:50:56,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,549 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=160, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,549 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,550 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,550 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656540) roll requested 2024-11-19T17:50:56,550 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656550 2024-11-19T17:50:56,558 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,558 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,558 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,558 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,558 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656540 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656550 2024-11-19T17:50:56,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,559 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=161, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,559 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,559 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,559 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,559 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656550) roll requested 2024-11-19T17:50:56,560 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656560 2024-11-19T17:50:56,567 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,567 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,567 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,567 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,567 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,567 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656550 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656560 2024-11-19T17:50:56,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,568 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=162, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,568 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,568 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,568 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656560) roll requested 2024-11-19T17:50:56,568 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656568 2024-11-19T17:50:56,576 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,576 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,576 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,576 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,576 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,577 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656560 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656568 2024-11-19T17:50:56,577 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,577 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,577 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=163, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,577 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656568) roll requested 2024-11-19T17:50:56,578 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656577 2024-11-19T17:50:56,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,584 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,584 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,584 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,584 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,585 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656568 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656577 2024-11-19T17:50:56,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,585 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=164, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,585 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,585 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656577) roll requested 2024-11-19T17:50:56,586 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656585 2024-11-19T17:50:56,596 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,596 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,596 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,596 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,596 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,596 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656577 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656585 2024-11-19T17:50:56,600 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,600 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=165, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,600 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,600 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,600 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,600 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656585) roll requested 2024-11-19T17:50:56,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656601 2024-11-19T17:50:56,606 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,606 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,606 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,606 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,606 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,606 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656585 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656601 2024-11-19T17:50:56,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,608 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=166, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,608 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,608 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656601) roll requested 2024-11-19T17:50:56,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656609 2024-11-19T17:50:56,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,614 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,614 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,614 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,614 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,614 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656601 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656609 2024-11-19T17:50:56,614 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,615 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=167, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,615 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,615 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,615 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,616 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656609) roll requested 2024-11-19T17:50:56,616 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656616 2024-11-19T17:50:56,620 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,620 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,620 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,620 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,621 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,621 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656609 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656616 2024-11-19T17:50:56,621 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,621 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=168, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,621 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,621 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,621 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656616) roll requested 2024-11-19T17:50:56,622 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656621 2024-11-19T17:50:56,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,626 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,627 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,627 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656616 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656621 2024-11-19T17:50:56,627 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,627 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=169, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,627 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,627 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,627 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,627 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656621) roll requested 2024-11-19T17:50:56,628 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656627 2024-11-19T17:50:56,632 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,632 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,632 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,632 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,633 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,633 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656621 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656627 2024-11-19T17:50:56,633 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,633 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=170, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,633 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,633 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,633 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,633 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656627) roll requested 2024-11-19T17:50:56,633 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656633 2024-11-19T17:50:56,639 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,639 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,639 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,639 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,639 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,639 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656627 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656633 2024-11-19T17:50:56,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,640 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=171, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,640 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,640 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,640 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,640 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656633) roll requested 2024-11-19T17:50:56,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656641 2024-11-19T17:50:56,648 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,648 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,648 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,648 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,648 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,648 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656633 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656641 2024-11-19T17:50:56,652 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,653 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=172, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,653 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,653 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,653 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,653 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656641) roll requested 2024-11-19T17:50:56,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656654 2024-11-19T17:50:56,659 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,659 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,659 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,659 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,659 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656641 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656654 2024-11-19T17:50:56,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,660 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=173, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,660 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,660 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656654) roll requested 2024-11-19T17:50:56,661 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656660 2024-11-19T17:50:56,666 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,666 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656654 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656660 2024-11-19T17:50:56,668 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,668 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=174, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,668 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,668 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,668 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,668 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656660) roll requested 2024-11-19T17:50:56,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656668 2024-11-19T17:50:56,674 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,674 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,674 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,674 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,674 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,674 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656660 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656668 2024-11-19T17:50:56,675 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,675 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=175, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,675 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,675 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,675 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,675 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656668) roll requested 2024-11-19T17:50:56,675 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656675 2024-11-19T17:50:56,681 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,681 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,681 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,681 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,681 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,681 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656668 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656675 2024-11-19T17:50:56,682 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,682 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=176, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,682 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,682 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,682 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,682 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656675) roll requested 2024-11-19T17:50:56,682 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656682 2024-11-19T17:50:56,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,695 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656675 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656682 2024-11-19T17:50:56,695 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,695 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=177, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,695 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,696 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,696 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656682) roll requested 2024-11-19T17:50:56,696 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656696 2024-11-19T17:50:56,702 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,702 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,702 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,702 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,702 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,702 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656682 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656696 2024-11-19T17:50:56,702 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,703 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=178, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,703 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,703 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,703 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,703 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656696) roll requested 2024-11-19T17:50:56,703 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656703 2024-11-19T17:50:56,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656696 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656703 2024-11-19T17:50:56,709 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,709 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=179, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,709 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,709 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656703) roll requested 2024-11-19T17:50:56,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656709 2024-11-19T17:50:56,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,714 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,714 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,714 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656703 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656709 2024-11-19T17:50:56,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,717 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=180, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,716 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,717 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,717 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,717 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656709) roll requested 2024-11-19T17:50:56,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656717 2024-11-19T17:50:56,721 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,722 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,722 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,722 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,722 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,722 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656709 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656717 2024-11-19T17:50:56,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,724 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=181, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,724 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,725 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,725 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656717) roll requested 2024-11-19T17:50:56,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656725 2024-11-19T17:50:56,730 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,730 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,730 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,730 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,730 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,730 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656717 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656725 2024-11-19T17:50:56,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,732 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=182, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,732 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,732 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,732 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,732 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656725) roll requested 2024-11-19T17:50:56,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656733 2024-11-19T17:50:56,740 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,740 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,740 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,740 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,740 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,740 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656725 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656733 2024-11-19T17:50:56,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,744 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=183, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,744 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,744 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656733) roll requested 2024-11-19T17:50:56,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656745 2024-11-19T17:50:56,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,750 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,750 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656733 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656745 2024-11-19T17:50:56,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,751 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=184, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,751 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,752 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656745) roll requested 2024-11-19T17:50:56,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656752 2024-11-19T17:50:56,757 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,757 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,757 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,757 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,757 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656745 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656752 2024-11-19T17:50:56,758 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,758 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=185, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,758 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,758 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,758 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,758 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656752) roll requested 2024-11-19T17:50:56,758 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656758 2024-11-19T17:50:56,768 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,769 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,769 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,769 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,769 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,769 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656752 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656758 2024-11-19T17:50:56,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,772 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=186, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,772 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,772 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656758) roll requested 2024-11-19T17:50:56,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656773 2024-11-19T17:50:56,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,777 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,777 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,777 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656758 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656773 2024-11-19T17:50:56,780 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,780 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=187, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,780 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,781 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,781 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,781 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656773) roll requested 2024-11-19T17:50:56,781 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656781 2024-11-19T17:50:56,786 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,786 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,786 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,786 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,786 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,786 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656773 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656781 2024-11-19T17:50:56,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,788 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=188, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,788 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,788 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,789 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656781) roll requested 2024-11-19T17:50:56,789 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656789 2024-11-19T17:50:56,793 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,793 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,794 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656781 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656789 2024-11-19T17:50:56,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,799 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=189, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,799 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,799 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656789) roll requested 2024-11-19T17:50:56,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656799 2024-11-19T17:50:56,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656789 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656799 2024-11-19T17:50:56,811 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,811 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,812 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=190, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,812 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656799) roll requested 2024-11-19T17:50:56,812 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656812 2024-11-19T17:50:56,822 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,823 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,823 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,823 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,823 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,823 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656799 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656812 2024-11-19T17:50:56,837 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,837 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=191, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,837 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,837 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,837 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656812) roll requested 2024-11-19T17:50:56,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656837 2024-11-19T17:50:56,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,852 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,852 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,852 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,852 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,852 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656812 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656837 2024-11-19T17:50:56,854 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,854 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=192, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,854 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,854 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,854 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,854 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656837) roll requested 2024-11-19T17:50:56,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656854 2024-11-19T17:50:56,869 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,869 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,869 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,869 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,870 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,870 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656837 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656854 2024-11-19T17:50:56,871 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,871 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=193, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,871 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,872 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,872 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,872 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656854) roll requested 2024-11-19T17:50:56,872 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656872 2024-11-19T17:50:56,890 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,890 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,890 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,891 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,891 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,891 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656854 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656872 2024-11-19T17:50:56,891 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,892 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=194, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,892 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,892 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,892 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,892 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656872) roll requested 2024-11-19T17:50:56,892 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656892 2024-11-19T17:50:56,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,900 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,901 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,901 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,901 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656872 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656892 2024-11-19T17:50:56,904 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,905 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=195, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,905 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,905 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,905 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,905 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656892) roll requested 2024-11-19T17:50:56,905 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656905 2024-11-19T17:50:56,911 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,912 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,912 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,912 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,912 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,912 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656892 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656905 2024-11-19T17:50:56,913 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,913 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=196, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,913 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,913 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,913 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,914 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656905) roll requested 2024-11-19T17:50:56,914 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656914 2024-11-19T17:50:56,920 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,921 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656905 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656914 2024-11-19T17:50:56,922 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,922 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=197, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,922 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,922 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,922 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,923 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656914) roll requested 2024-11-19T17:50:56,923 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656923 2024-11-19T17:50:56,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,932 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,933 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656914 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656923 2024-11-19T17:50:56,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:56,960 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=198, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:56,960 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:56,961 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,962 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:56,962 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656923) roll requested 2024-11-19T17:50:56,962 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038656962 2024-11-19T17:50:56,981 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,982 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,983 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:56,983 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656923 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656962 2024-11-19T17:50:57,013 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,013 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=199, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,013 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,013 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,013 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,013 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038656962) roll requested 2024-11-19T17:50:57,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657014 2024-11-19T17:50:57,038 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,038 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,038 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,038 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,038 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,038 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038656962 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657014 2024-11-19T17:50:57,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,044 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=200, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,045 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,045 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,045 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657014) roll requested 2024-11-19T17:50:57,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657045 2024-11-19T17:50:57,073 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,073 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,073 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,073 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,073 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,073 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657014 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657045 2024-11-19T17:50:57,081 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,081 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=201, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,081 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,081 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,081 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,081 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657045) roll requested 2024-11-19T17:50:57,081 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657081 2024-11-19T17:50:57,095 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,095 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657045 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657081 2024-11-19T17:50:57,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,097 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=202, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,097 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,097 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,097 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657081) roll requested 2024-11-19T17:50:57,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657097 2024-11-19T17:50:57,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657081 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657097 2024-11-19T17:50:57,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,116 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=203, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,117 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,117 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,117 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,117 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657097) roll requested 2024-11-19T17:50:57,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657117 2024-11-19T17:50:57,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,129 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,129 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,129 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,129 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657097 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657117 2024-11-19T17:50:57,132 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,132 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,133 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=204, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,133 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,133 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657117) roll requested 2024-11-19T17:50:57,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657133 2024-11-19T17:50:57,143 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,143 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,143 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,144 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,144 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,144 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657117 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657133 2024-11-19T17:50:57,144 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,144 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=205, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,145 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,145 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657133) roll requested 2024-11-19T17:50:57,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657145 2024-11-19T17:50:57,150 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,150 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,150 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,151 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657133 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657145 2024-11-19T17:50:57,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,152 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=206, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,152 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,152 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,152 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657145) roll requested 2024-11-19T17:50:57,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657152 2024-11-19T17:50:57,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657145 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657152 2024-11-19T17:50:57,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,159 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=207, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,159 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,159 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,159 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657152) roll requested 2024-11-19T17:50:57,159 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657159 2024-11-19T17:50:57,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657152 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657159 2024-11-19T17:50:57,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,172 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=208, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,172 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,172 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657159) roll requested 2024-11-19T17:50:57,172 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657172 2024-11-19T17:50:57,185 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,185 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,185 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,185 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,185 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657159 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657172 2024-11-19T17:50:57,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,188 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=209, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,188 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,188 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657172) roll requested 2024-11-19T17:50:57,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657189 2024-11-19T17:50:57,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,194 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,194 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,194 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657172 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657189 2024-11-19T17:50:57,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,195 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=210, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,195 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,196 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,196 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657189) roll requested 2024-11-19T17:50:57,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657196 2024-11-19T17:50:57,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,202 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657189 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657196 2024-11-19T17:50:57,202 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,202 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=211, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,202 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,202 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,202 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,202 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657196) roll requested 2024-11-19T17:50:57,203 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657202 2024-11-19T17:50:57,216 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,216 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,216 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,216 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,217 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,217 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657196 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657202 2024-11-19T17:50:57,217 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,218 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=212, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,218 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,218 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,220 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,220 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657202) roll requested 2024-11-19T17:50:57,220 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657220 2024-11-19T17:50:57,227 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,227 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657202 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657220 2024-11-19T17:50:57,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,229 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=213, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,229 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,236 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,236 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,236 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657220) roll requested 2024-11-19T17:50:57,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657236 2024-11-19T17:50:57,243 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,243 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,243 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,243 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,243 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,243 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657220 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657236 2024-11-19T17:50:57,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,244 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=214, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,244 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,247 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,247 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,247 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657236) roll requested 2024-11-19T17:50:57,247 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657247 2024-11-19T17:50:57,259 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,259 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,259 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,259 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,260 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,260 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657236 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657247 2024-11-19T17:50:57,260 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,260 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=215, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,261 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,261 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657247) roll requested 2024-11-19T17:50:57,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657261 2024-11-19T17:50:57,267 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,267 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,267 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,267 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,267 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,267 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657247 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657261 2024-11-19T17:50:57,269 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,269 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,269 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=216, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,269 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,270 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,270 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657261) roll requested 2024-11-19T17:50:57,270 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657270 2024-11-19T17:50:57,280 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,280 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,280 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,280 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,280 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657261 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657270 2024-11-19T17:50:57,281 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,281 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,281 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=217, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,281 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,282 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,282 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657270) roll requested 2024-11-19T17:50:57,282 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657282 2024-11-19T17:50:57,291 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,291 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657270 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657282 2024-11-19T17:50:57,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,292 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=218, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,292 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,292 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657282) roll requested 2024-11-19T17:50:57,292 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657292 2024-11-19T17:50:57,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657282 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657292 2024-11-19T17:50:57,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,301 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=219, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,301 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,301 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657292) roll requested 2024-11-19T17:50:57,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657301 2024-11-19T17:50:57,308 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,308 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,308 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,308 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,308 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,308 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657292 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657301 2024-11-19T17:50:57,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,312 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=220, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,312 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,312 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657301) roll requested 2024-11-19T17:50:57,313 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657312 2024-11-19T17:50:57,318 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,318 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,318 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,318 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,318 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657301 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657312 2024-11-19T17:50:57,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,320 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=221, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,320 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,320 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657312) roll requested 2024-11-19T17:50:57,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657320 2024-11-19T17:50:57,325 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,326 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,326 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,326 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,326 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,326 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657312 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657320 2024-11-19T17:50:57,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,328 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=222, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,328 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,328 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,328 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,328 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657320) roll requested 2024-11-19T17:50:57,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657328 2024-11-19T17:50:57,333 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,334 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,334 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,334 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,334 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,334 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657320 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657328 2024-11-19T17:50:57,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,336 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=223, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,336 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,336 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,337 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657328) roll requested 2024-11-19T17:50:57,337 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657337 2024-11-19T17:50:57,344 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,344 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,344 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,344 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,344 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,345 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657328 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657337 2024-11-19T17:50:57,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,348 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=224, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,348 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,348 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,348 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,348 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657337) roll requested 2024-11-19T17:50:57,348 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657348 2024-11-19T17:50:57,356 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,356 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,356 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,356 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,357 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,357 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657337 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657348 2024-11-19T17:50:57,358 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,358 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=225, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,358 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,358 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,358 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,358 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657348) roll requested 2024-11-19T17:50:57,359 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657358 2024-11-19T17:50:57,366 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,366 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,366 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,366 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,366 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,366 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657348 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657358 2024-11-19T17:50:57,367 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,367 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,367 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=226, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,367 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,367 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,367 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657358) roll requested 2024-11-19T17:50:57,367 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657367 2024-11-19T17:50:57,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,384 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,384 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657358 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657367 2024-11-19T17:50:57,385 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,385 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=227, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,385 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,385 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,385 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,385 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657367) roll requested 2024-11-19T17:50:57,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657385 2024-11-19T17:50:57,391 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,391 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,391 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,391 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,391 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,392 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657367 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657385 2024-11-19T17:50:57,394 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,394 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,394 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=228, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,394 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,394 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657385) roll requested 2024-11-19T17:50:57,395 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657394 2024-11-19T17:50:57,400 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,400 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,400 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,401 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,401 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,401 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657385 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657394 2024-11-19T17:50:57,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,408 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=229, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,408 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,408 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,408 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657394) roll requested 2024-11-19T17:50:57,408 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657408 2024-11-19T17:50:57,413 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,413 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,414 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657394 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657408 2024-11-19T17:50:57,414 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,414 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=230, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,414 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,414 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,414 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657408) roll requested 2024-11-19T17:50:57,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657415 2024-11-19T17:50:57,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657408 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657415 2024-11-19T17:50:57,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,420 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=231, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,420 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,420 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657415) roll requested 2024-11-19T17:50:57,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657420 2024-11-19T17:50:57,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,428 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,430 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657415 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657420 2024-11-19T17:50:57,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,433 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,433 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,433 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=232, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,433 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657420) roll requested 2024-11-19T17:50:57,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657433 2024-11-19T17:50:57,454 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,454 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,455 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,455 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,455 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657420 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657433 2024-11-19T17:50:57,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,460 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,460 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=233, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,461 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657433) roll requested 2024-11-19T17:50:57,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657461 2024-11-19T17:50:57,469 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,469 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,469 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,469 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,470 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,470 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657433 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657461 2024-11-19T17:50:57,479 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,479 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=234, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,479 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,479 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,479 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,479 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657461) roll requested 2024-11-19T17:50:57,480 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657480 2024-11-19T17:50:57,488 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,488 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,488 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,488 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657461 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657480 2024-11-19T17:50:57,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,490 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,490 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=235, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,490 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,491 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657480) roll requested 2024-11-19T17:50:57,491 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657491 2024-11-19T17:50:57,498 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,498 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657480 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657491 2024-11-19T17:50:57,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,500 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=236, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,500 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,501 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657491) roll requested 2024-11-19T17:50:57,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657501 2024-11-19T17:50:57,508 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,508 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,508 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,509 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,509 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,509 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657491 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657501 2024-11-19T17:50:57,510 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,510 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=237, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,510 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,510 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,510 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,510 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657501) roll requested 2024-11-19T17:50:57,511 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657510 2024-11-19T17:50:57,518 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,519 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,519 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,519 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,519 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,519 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657501 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657510 2024-11-19T17:50:57,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,520 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=238, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,521 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,521 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657510) roll requested 2024-11-19T17:50:57,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657521 2024-11-19T17:50:57,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657510 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657521 2024-11-19T17:50:57,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,531 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,531 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=239, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,532 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,532 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657521) roll requested 2024-11-19T17:50:57,532 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657532 2024-11-19T17:50:57,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,549 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,550 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,550 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657521 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657532 2024-11-19T17:50:57,550 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,550 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=240, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,550 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,550 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,550 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,550 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657532) roll requested 2024-11-19T17:50:57,551 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657551 2024-11-19T17:50:57,560 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,560 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,561 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,561 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,561 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,561 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657532 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657551 2024-11-19T17:50:57,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,564 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,565 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=241, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,565 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657551) roll requested 2024-11-19T17:50:57,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657565 2024-11-19T17:50:57,572 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,572 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,572 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,572 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,572 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,572 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657551 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657565 2024-11-19T17:50:57,573 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,573 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,573 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=242, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,573 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,573 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,573 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657565) roll requested 2024-11-19T17:50:57,574 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657573 2024-11-19T17:50:57,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657565 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657573 2024-11-19T17:50:57,583 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,584 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,584 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=243, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,584 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657573) roll requested 2024-11-19T17:50:57,584 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657584 2024-11-19T17:50:57,589 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,589 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,589 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,589 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,589 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,589 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657573 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657584 2024-11-19T17:50:57,589 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,590 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=244, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,590 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,590 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,590 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,590 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657584) roll requested 2024-11-19T17:50:57,590 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657590 2024-11-19T17:50:57,595 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,595 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,595 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,595 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,595 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,596 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657584 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657590 2024-11-19T17:50:57,596 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,596 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,596 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=245, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,596 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,596 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,597 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657590) roll requested 2024-11-19T17:50:57,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657597 2024-11-19T17:50:57,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,601 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657590 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657597 2024-11-19T17:50:57,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,607 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,607 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=246, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,607 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,607 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,607 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657597) roll requested 2024-11-19T17:50:57,607 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657607 2024-11-19T17:50:57,612 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,612 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,612 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,612 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,612 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,612 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657597 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657607 2024-11-19T17:50:57,613 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,613 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=247, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,613 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,613 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,613 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,613 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657607) roll requested 2024-11-19T17:50:57,613 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657613 2024-11-19T17:50:57,617 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,617 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657607 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657613 2024-11-19T17:50:57,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,620 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,620 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=248, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,620 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,620 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657613) roll requested 2024-11-19T17:50:57,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657620 2024-11-19T17:50:57,625 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,625 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,625 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,625 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,626 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,626 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657613 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657620 2024-11-19T17:50:57,626 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,626 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=249, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,626 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,626 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,627 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,627 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657620) roll requested 2024-11-19T17:50:57,627 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657627 2024-11-19T17:50:57,632 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,632 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,632 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,632 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,632 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,632 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657620 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657627 2024-11-19T17:50:57,633 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,633 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,633 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=250, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,633 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,633 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,633 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657627) roll requested 2024-11-19T17:50:57,633 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657633 2024-11-19T17:50:57,637 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,637 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,637 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,637 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,637 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,637 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657627 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657633 2024-11-19T17:50:57,638 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,638 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,638 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,638 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=251, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,638 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,638 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657633) roll requested 2024-11-19T17:50:57,638 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657638 2024-11-19T17:50:57,643 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,643 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,643 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,644 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,644 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,644 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657633 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657638 2024-11-19T17:50:57,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,644 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,644 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=252, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,645 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657638) roll requested 2024-11-19T17:50:57,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657645 2024-11-19T17:50:57,650 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,650 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,650 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,650 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,650 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657638 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657645 2024-11-19T17:50:57,651 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,651 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,651 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=253, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,651 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,651 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657645) roll requested 2024-11-19T17:50:57,651 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657651 2024-11-19T17:50:57,655 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,655 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,655 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,655 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,656 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657645 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657651 2024-11-19T17:50:57,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,656 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=254, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,656 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,656 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657651) roll requested 2024-11-19T17:50:57,656 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657656 2024-11-19T17:50:57,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657651 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657656 2024-11-19T17:50:57,662 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,662 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,662 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=255, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,662 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,662 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,662 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657656) roll requested 2024-11-19T17:50:57,662 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657662 2024-11-19T17:50:57,668 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,668 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,668 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,668 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,668 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,668 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657656 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657662 2024-11-19T17:50:57,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,672 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=256, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,672 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,672 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,672 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657662) roll requested 2024-11-19T17:50:57,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657672 2024-11-19T17:50:57,678 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,678 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,678 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,678 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,678 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,678 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657662 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657672 2024-11-19T17:50:57,679 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,679 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,679 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=257, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,679 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,679 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,679 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657672) roll requested 2024-11-19T17:50:57,679 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657679 2024-11-19T17:50:57,684 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,684 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,684 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,684 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,684 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,684 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657672 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657679 2024-11-19T17:50:57,685 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,685 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,685 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=258, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,685 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,685 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,685 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657679) roll requested 2024-11-19T17:50:57,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657685 2024-11-19T17:50:57,689 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,689 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,690 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,690 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,690 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,690 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657679 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657685 2024-11-19T17:50:57,690 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,690 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,690 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=259, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,690 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,691 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,691 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657685) roll requested 2024-11-19T17:50:57,691 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657691 2024-11-19T17:50:57,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657685 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657691 2024-11-19T17:50:57,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,697 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=260, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,697 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,697 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657691) roll requested 2024-11-19T17:50:57,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657697 2024-11-19T17:50:57,702 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,702 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,702 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,702 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,702 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,702 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657691 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657697 2024-11-19T17:50:57,703 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,703 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,703 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=261, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,703 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,703 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,703 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657697) roll requested 2024-11-19T17:50:57,703 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657703 2024-11-19T17:50:57,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657697 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657703 2024-11-19T17:50:57,709 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,709 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,709 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=262, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,709 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657703) roll requested 2024-11-19T17:50:57,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657709 2024-11-19T17:50:57,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,714 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,714 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,714 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657703 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657709 2024-11-19T17:50:57,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,716 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,716 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=263, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,716 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,716 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,716 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657709) roll requested 2024-11-19T17:50:57,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657717 2024-11-19T17:50:57,721 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,721 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,721 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,721 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,721 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,721 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657709 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657717 2024-11-19T17:50:57,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,724 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=264, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,724 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,725 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657717) roll requested 2024-11-19T17:50:57,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657725 2024-11-19T17:50:57,729 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,729 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,729 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,729 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,729 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,729 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657717 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657725 2024-11-19T17:50:57,730 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,730 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=265, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,730 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,730 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,730 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,730 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657725) roll requested 2024-11-19T17:50:57,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657730 2024-11-19T17:50:57,734 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,734 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657725 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657730 2024-11-19T17:50:57,735 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,735 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=266, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,735 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,735 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,735 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,735 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657730) roll requested 2024-11-19T17:50:57,735 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657735 2024-11-19T17:50:57,740 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,740 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,740 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,740 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,740 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,740 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657730 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657735 2024-11-19T17:50:57,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,744 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=267, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,744 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,744 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657735) roll requested 2024-11-19T17:50:57,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657745 2024-11-19T17:50:57,749 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,749 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,749 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,749 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657735 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657745 2024-11-19T17:50:57,750 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,750 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=268, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,750 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,750 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,750 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,750 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657745) roll requested 2024-11-19T17:50:57,750 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657750 2024-11-19T17:50:57,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657745 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657750 2024-11-19T17:50:57,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,756 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,756 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=269, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,756 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657750) roll requested 2024-11-19T17:50:57,756 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657756 2024-11-19T17:50:57,760 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,761 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,761 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,761 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,761 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,761 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657750 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657756 2024-11-19T17:50:57,761 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,761 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=270, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,761 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,762 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,762 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,762 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657756) roll requested 2024-11-19T17:50:57,762 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657762 2024-11-19T17:50:57,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,766 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657756 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657762 2024-11-19T17:50:57,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,772 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,772 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=271, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,773 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657762) roll requested 2024-11-19T17:50:57,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657773 2024-11-19T17:50:57,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,777 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657762 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657773 2024-11-19T17:50:57,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,778 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=272, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,778 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,778 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,778 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657773) roll requested 2024-11-19T17:50:57,779 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657779 2024-11-19T17:50:57,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,784 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657773 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657779 2024-11-19T17:50:57,785 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,785 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,785 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=273, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,785 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,785 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657779) roll requested 2024-11-19T17:50:57,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657785 2024-11-19T17:50:57,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,790 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,790 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657779 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657785 2024-11-19T17:50:57,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,791 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=274, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,791 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,791 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657785) roll requested 2024-11-19T17:50:57,791 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657791 2024-11-19T17:50:57,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,796 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,796 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,796 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,796 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,796 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657785 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657791 2024-11-19T17:50:57,797 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,797 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=275, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,797 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,797 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,797 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,797 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657791) roll requested 2024-11-19T17:50:57,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657797 2024-11-19T17:50:57,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657791 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657797 2024-11-19T17:50:57,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,804 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=276, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,804 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,804 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657797) roll requested 2024-11-19T17:50:57,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657804 2024-11-19T17:50:57,809 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,810 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,810 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,810 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,810 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,810 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657797 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657804 2024-11-19T17:50:57,810 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,810 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=277, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,810 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,810 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,811 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,811 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657804) roll requested 2024-11-19T17:50:57,811 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657811 2024-11-19T17:50:57,815 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,815 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,815 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,815 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,815 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,815 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657804 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657811 2024-11-19T17:50:57,816 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,816 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=278, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,816 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,816 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,816 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,816 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657811) roll requested 2024-11-19T17:50:57,816 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657816 2024-11-19T17:50:57,821 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,821 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657811 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657816 2024-11-19T17:50:57,822 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,822 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,822 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=279, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,822 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,822 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657816) roll requested 2024-11-19T17:50:57,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657822 2024-11-19T17:50:57,826 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,826 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,826 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657816 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657822 2024-11-19T17:50:57,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,827 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,827 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=280, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,827 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,827 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657822) roll requested 2024-11-19T17:50:57,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657828 2024-11-19T17:50:57,832 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,832 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,832 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,832 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,832 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,832 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657822 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657828 2024-11-19T17:50:57,833 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,833 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,833 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=281, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,833 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,833 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,833 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657828) roll requested 2024-11-19T17:50:57,833 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657833 2024-11-19T17:50:57,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,838 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,838 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657828 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657833 2024-11-19T17:50:57,839 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,839 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,839 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=282, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,839 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,839 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,839 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657833) roll requested 2024-11-19T17:50:57,839 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657839 2024-11-19T17:50:57,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,844 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,844 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,844 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,844 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,844 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657833 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657839 2024-11-19T17:50:57,844 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,844 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,845 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,845 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=283, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,845 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657839) roll requested 2024-11-19T17:50:57,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657845 2024-11-19T17:50:57,850 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,850 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,850 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,850 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,850 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,850 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657839 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657845 2024-11-19T17:50:57,851 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,851 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=284, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,851 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,851 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,851 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,851 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657845) roll requested 2024-11-19T17:50:57,851 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657851 2024-11-19T17:50:57,857 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,857 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,858 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,858 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,858 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657845 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657851 2024-11-19T17:50:57,858 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,859 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,859 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=285, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,859 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,859 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,859 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657851) roll requested 2024-11-19T17:50:57,859 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657859 2024-11-19T17:50:57,865 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,865 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,866 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,866 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657851 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657859 2024-11-19T17:50:57,866 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,866 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,867 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=286, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,867 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,867 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657859) roll requested 2024-11-19T17:50:57,867 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657867 2024-11-19T17:50:57,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,872 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,872 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,872 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,872 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657859 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657867 2024-11-19T17:50:57,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,873 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,873 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=287, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,873 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,873 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657867) roll requested 2024-11-19T17:50:57,873 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657873 2024-11-19T17:50:57,879 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,879 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,879 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,879 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,879 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,879 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657867 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657873 2024-11-19T17:50:57,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,880 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,880 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=288, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,880 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,881 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657873) roll requested 2024-11-19T17:50:57,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657881 2024-11-19T17:50:57,887 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,887 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,887 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,887 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,887 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657873 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657881 2024-11-19T17:50:57,888 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,888 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,888 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,888 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=289, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,889 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,889 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657881) roll requested 2024-11-19T17:50:57,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657889 2024-11-19T17:50:57,896 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,896 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,897 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,897 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,897 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,897 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657881 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657889 2024-11-19T17:50:57,898 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,898 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,898 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,898 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=290, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,898 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,898 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657889) roll requested 2024-11-19T17:50:57,899 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657898 2024-11-19T17:50:57,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,906 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657889 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657898 2024-11-19T17:50:57,907 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,908 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,908 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=291, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,908 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,908 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657898) roll requested 2024-11-19T17:50:57,908 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657908 2024-11-19T17:50:57,915 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,915 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,915 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,915 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,915 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,915 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657898 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657908 2024-11-19T17:50:57,915 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,916 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,916 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=292, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,916 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,916 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,916 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657908) roll requested 2024-11-19T17:50:57,916 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657916 2024-11-19T17:50:57,922 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,922 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,922 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,922 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,922 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,922 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657908 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657916 2024-11-19T17:50:57,923 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,923 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,923 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,923 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=293, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,923 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,923 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657916) roll requested 2024-11-19T17:50:57,924 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657924 2024-11-19T17:50:57,929 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,929 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,929 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,929 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,929 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,929 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657916 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657924 2024-11-19T17:50:57,930 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,930 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,930 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=294, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,930 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,930 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,930 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657924) roll requested 2024-11-19T17:50:57,930 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657930 2024-11-19T17:50:57,935 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,935 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,935 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,935 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,935 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,935 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657924 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657930 2024-11-19T17:50:57,936 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,936 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,936 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=295, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,936 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,936 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,936 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657930) roll requested 2024-11-19T17:50:57,936 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657936 2024-11-19T17:50:57,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,942 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,942 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,942 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,942 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,942 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657930 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657936 2024-11-19T17:50:57,943 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,943 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,943 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=296, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,943 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,943 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,943 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657936) roll requested 2024-11-19T17:50:57,943 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657943 2024-11-19T17:50:57,950 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,950 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657936 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657943 2024-11-19T17:50:57,951 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,951 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,951 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=297, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,952 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657943) roll requested 2024-11-19T17:50:57,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657952 2024-11-19T17:50:57,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,958 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,958 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,958 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,958 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,958 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657943 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657952 2024-11-19T17:50:57,959 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,959 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,959 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=298, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,959 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,959 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,959 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657952) roll requested 2024-11-19T17:50:57,959 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657959 2024-11-19T17:50:57,964 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,964 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,964 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,964 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,964 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,964 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657952 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657959 2024-11-19T17:50:57,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,965 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,965 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=299, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,965 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657959) roll requested 2024-11-19T17:50:57,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657965 2024-11-19T17:50:57,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657959 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657965 2024-11-19T17:50:57,977 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,977 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,977 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=300, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,978 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,978 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,978 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657965) roll requested 2024-11-19T17:50:57,978 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657978 2024-11-19T17:50:57,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657965 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657978 2024-11-19T17:50:57,993 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,993 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,993 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=301, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,993 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657978) roll requested 2024-11-19T17:50:57,994 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657994 2024-11-19T17:50:57,998 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,998 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:57,998 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657978 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657994 2024-11-19T17:50:57,999 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:57,999 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,999 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=302, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:57,999 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:57,999 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:57,999 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657994) roll requested 2024-11-19T17:50:57,999 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038657999 2024-11-19T17:50:58,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,005 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,005 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657994 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657999 2024-11-19T17:50:58,006 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,006 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,007 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=303, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,007 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038657999) roll requested 2024-11-19T17:50:58,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658007 2024-11-19T17:50:58,012 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,012 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,012 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,012 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,012 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,012 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038657999 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658007 2024-11-19T17:50:58,013 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,013 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,013 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=304, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,013 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,013 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,013 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658007) roll requested 2024-11-19T17:50:58,013 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658013 2024-11-19T17:50:58,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,018 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,018 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658007 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658013 2024-11-19T17:50:58,019 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,019 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,019 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=305, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,019 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,019 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,019 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658013) roll requested 2024-11-19T17:50:58,019 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658019 2024-11-19T17:50:58,025 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,025 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,025 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,025 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,025 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,025 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658013 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658019 2024-11-19T17:50:58,026 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,026 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,026 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,026 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=306, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,026 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,026 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658019) roll requested 2024-11-19T17:50:58,026 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658026 2024-11-19T17:50:58,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658019 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658026 2024-11-19T17:50:58,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,032 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,032 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=307, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,032 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658026) roll requested 2024-11-19T17:50:58,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658032 2024-11-19T17:50:58,037 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,037 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,038 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,038 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,038 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,038 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658026 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658032 2024-11-19T17:50:58,038 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,038 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,038 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=308, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,038 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,039 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,039 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658032) roll requested 2024-11-19T17:50:58,039 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658039 2024-11-19T17:50:58,044 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,044 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,044 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,045 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,045 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,045 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658032 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658039 2024-11-19T17:50:58,045 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,045 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,045 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=309, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,045 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,045 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,046 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658039) roll requested 2024-11-19T17:50:58,046 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658046 2024-11-19T17:50:58,050 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,050 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,050 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,051 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,051 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,051 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658039 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658046 2024-11-19T17:50:58,051 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,051 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,051 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,051 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=310, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,052 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658046) roll requested 2024-11-19T17:50:58,052 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658052 2024-11-19T17:50:58,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,056 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,056 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,056 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658046 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658052 2024-11-19T17:50:58,057 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,057 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,057 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=311, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,057 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,057 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658052) roll requested 2024-11-19T17:50:58,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658057 2024-11-19T17:50:58,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,061 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,062 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,062 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,062 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,062 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658052 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658057 2024-11-19T17:50:58,062 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,062 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,062 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,062 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=312, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,063 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,063 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658057) roll requested 2024-11-19T17:50:58,063 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658063 2024-11-19T17:50:58,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,067 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658057 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658063 2024-11-19T17:50:58,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,068 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,068 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,068 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=313, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,068 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,068 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658063) roll requested 2024-11-19T17:50:58,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658068 2024-11-19T17:50:58,073 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,073 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,073 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,073 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,073 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,073 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658063 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658068 2024-11-19T17:50:58,074 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,074 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,074 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=314, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,074 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,074 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,074 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658068) roll requested 2024-11-19T17:50:58,074 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658074 2024-11-19T17:50:58,079 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,079 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,079 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,079 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,079 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658068 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658074 2024-11-19T17:50:58,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,080 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,080 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=315, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,080 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,080 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658074) roll requested 2024-11-19T17:50:58,080 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658080 2024-11-19T17:50:58,085 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,085 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,085 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,085 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,085 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658074 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658080 2024-11-19T17:50:58,085 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,085 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,086 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=316, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,086 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,086 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,086 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658080) roll requested 2024-11-19T17:50:58,086 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658086 2024-11-19T17:50:58,090 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,090 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,090 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,090 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,090 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,090 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658080 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658086 2024-11-19T17:50:58,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,091 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,091 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=317, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,091 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,091 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,091 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658086) roll requested 2024-11-19T17:50:58,091 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658091 2024-11-19T17:50:58,095 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,095 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658086 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658091 2024-11-19T17:50:58,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,096 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,096 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=318, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,096 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658091) roll requested 2024-11-19T17:50:58,096 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658096 2024-11-19T17:50:58,101 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,101 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,101 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,101 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,101 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,101 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658091 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658096 2024-11-19T17:50:58,104 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,104 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,104 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=319, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,105 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658096) roll requested 2024-11-19T17:50:58,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658105 2024-11-19T17:50:58,110 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,110 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,110 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,110 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,110 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,110 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658096 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658105 2024-11-19T17:50:58,110 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,110 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,110 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=320, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,110 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,111 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,111 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658105) roll requested 2024-11-19T17:50:58,111 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658111 2024-11-19T17:50:58,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658105 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658111 2024-11-19T17:50:58,119 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,119 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,119 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=321, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,119 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,119 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,119 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658111) roll requested 2024-11-19T17:50:58,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658119 2024-11-19T17:50:58,126 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,126 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,126 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,126 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,126 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,127 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658111 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658119 2024-11-19T17:50:58,127 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,127 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,127 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=322, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,127 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,127 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,127 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658119) roll requested 2024-11-19T17:50:58,128 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658127 2024-11-19T17:50:58,133 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,133 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,133 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,133 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,133 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,133 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658119 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658127 2024-11-19T17:50:58,134 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,134 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,134 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,134 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=323, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,134 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,134 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658127) roll requested 2024-11-19T17:50:58,134 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658134 2024-11-19T17:50:58,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,143 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,143 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658127 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658134 2024-11-19T17:50:58,143 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,143 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,143 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=324, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,143 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,143 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,144 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658134) roll requested 2024-11-19T17:50:58,144 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658144 2024-11-19T17:50:58,148 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,148 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,148 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,148 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,148 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,148 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658134 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658144 2024-11-19T17:50:58,149 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,149 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,149 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=325, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,149 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658144) roll requested 2024-11-19T17:50:58,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658149 2024-11-19T17:50:58,154 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,154 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,154 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,154 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,154 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,154 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658144 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658149 2024-11-19T17:50:58,155 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,155 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=326, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,155 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,155 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,155 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,155 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658149) roll requested 2024-11-19T17:50:58,155 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658155 2024-11-19T17:50:58,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,160 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,160 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658149 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658155 2024-11-19T17:50:58,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,161 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=327, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,161 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,161 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658155) roll requested 2024-11-19T17:50:58,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658161 2024-11-19T17:50:58,166 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,166 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,166 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,166 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,166 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,166 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658155 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658161 2024-11-19T17:50:58,167 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,167 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,167 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=328, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,167 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,167 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,167 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658161) roll requested 2024-11-19T17:50:58,168 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658167 2024-11-19T17:50:58,172 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,172 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,172 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,172 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,172 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,173 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658161 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658167 2024-11-19T17:50:58,173 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,173 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=329, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,173 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,173 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,173 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658167) roll requested 2024-11-19T17:50:58,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658173 2024-11-19T17:50:58,178 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658167 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658173 2024-11-19T17:50:58,179 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,180 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=330, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,180 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,180 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658173) roll requested 2024-11-19T17:50:58,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658180 2024-11-19T17:50:58,185 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,185 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,185 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,185 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,185 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658173 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658180 2024-11-19T17:50:58,186 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,186 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=331, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,186 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,186 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,186 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,186 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658180) roll requested 2024-11-19T17:50:58,186 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658186 2024-11-19T17:50:58,191 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,191 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,191 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,191 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,192 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,192 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658180 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658186 2024-11-19T17:50:58,192 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,193 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,193 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=332, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,193 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,193 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,193 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658186) roll requested 2024-11-19T17:50:58,193 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658193 2024-11-19T17:50:58,198 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,198 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,198 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,198 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,198 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,198 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658186 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658193 2024-11-19T17:50:58,199 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,199 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=333, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,199 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,199 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,199 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,199 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658193) roll requested 2024-11-19T17:50:58,199 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658199 2024-11-19T17:50:58,203 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,203 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658193 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658199 2024-11-19T17:50:58,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,204 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,204 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=334, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,205 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658199) roll requested 2024-11-19T17:50:58,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658205 2024-11-19T17:50:58,209 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,209 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658199 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658205 2024-11-19T17:50:58,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,210 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,210 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=335, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,210 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,210 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658205) roll requested 2024-11-19T17:50:58,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658210 2024-11-19T17:50:58,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658205 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658210 2024-11-19T17:50:58,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,216 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,216 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=336, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,216 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658210) roll requested 2024-11-19T17:50:58,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658216 2024-11-19T17:50:58,221 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,221 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,221 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,221 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658210 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658216 2024-11-19T17:50:58,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,225 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=337, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,224 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,225 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658216) roll requested 2024-11-19T17:50:58,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658225 2024-11-19T17:50:58,229 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,229 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,229 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,229 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,229 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,229 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658216 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658225 2024-11-19T17:50:58,230 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,230 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,230 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=338, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,230 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,230 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,230 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658225) roll requested 2024-11-19T17:50:58,230 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658230 2024-11-19T17:50:58,234 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,234 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,235 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,235 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,235 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,235 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658225 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658230 2024-11-19T17:50:58,235 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,235 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,235 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=339, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,235 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,236 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,236 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658230) roll requested 2024-11-19T17:50:58,236 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658236 2024-11-19T17:50:58,240 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,240 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,240 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,240 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,240 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,240 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658230 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658236 2024-11-19T17:50:58,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,241 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,241 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=340, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,241 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,241 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,241 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658236) roll requested 2024-11-19T17:50:58,241 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658241 2024-11-19T17:50:58,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658236 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658241 2024-11-19T17:50:58,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,246 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,246 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=341, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,246 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658241) roll requested 2024-11-19T17:50:58,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658246 2024-11-19T17:50:58,251 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,251 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,251 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,251 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658241 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658246 2024-11-19T17:50:58,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,252 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,252 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=342, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,252 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,252 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658246) roll requested 2024-11-19T17:50:58,252 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658252 2024-11-19T17:50:58,257 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,257 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,257 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658246 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658252 2024-11-19T17:50:58,258 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,258 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,258 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=343, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,258 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,258 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658252) roll requested 2024-11-19T17:50:58,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658259 2024-11-19T17:50:58,264 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,264 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,264 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,264 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,264 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,264 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658252 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658259 2024-11-19T17:50:58,265 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,265 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,265 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,265 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=344, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,266 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,266 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658259) roll requested 2024-11-19T17:50:58,266 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658266 2024-11-19T17:50:58,273 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,273 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,273 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,273 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,273 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,273 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658259 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658266 2024-11-19T17:50:58,274 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,274 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,274 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=345, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,274 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,274 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,274 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658266) roll requested 2024-11-19T17:50:58,275 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658275 2024-11-19T17:50:58,279 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,279 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,279 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,279 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,279 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,280 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658266 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658275 2024-11-19T17:50:58,280 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,280 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,280 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=346, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,280 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,280 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,280 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658275) roll requested 2024-11-19T17:50:58,280 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658280 2024-11-19T17:50:58,285 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,285 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,285 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,285 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,285 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,285 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658275 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658280 2024-11-19T17:50:58,286 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,286 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,286 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=347, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,286 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,286 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,286 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658280) roll requested 2024-11-19T17:50:58,286 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658286 2024-11-19T17:50:58,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658280 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658286 2024-11-19T17:50:58,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,293 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,293 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=348, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,293 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658286) roll requested 2024-11-19T17:50:58,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658293 2024-11-19T17:50:58,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,299 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658286 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658293 2024-11-19T17:50:58,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,300 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,300 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=349, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,300 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,300 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658293) roll requested 2024-11-19T17:50:58,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658301 2024-11-19T17:50:58,306 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,307 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,307 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,307 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,307 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658293 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658301 2024-11-19T17:50:58,307 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,308 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,308 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=350, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,308 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658301) roll requested 2024-11-19T17:50:58,308 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658308 2024-11-19T17:50:58,314 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,314 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,314 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,314 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,314 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,314 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658301 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658308 2024-11-19T17:50:58,315 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,315 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,315 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=351, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,315 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,315 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,315 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658308) roll requested 2024-11-19T17:50:58,316 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658316 2024-11-19T17:50:58,323 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,323 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,323 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,323 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,323 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,323 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658308 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658316 2024-11-19T17:50:58,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,324 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,324 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=352, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,324 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,325 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658316) roll requested 2024-11-19T17:50:58,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658325 2024-11-19T17:50:58,332 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,332 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,332 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,332 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,332 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,332 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658316 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658325 2024-11-19T17:50:58,333 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,333 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,333 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=353, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,333 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,333 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,333 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658325) roll requested 2024-11-19T17:50:58,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658333 2024-11-19T17:50:58,343 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,344 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,344 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,344 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658325 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658333 2024-11-19T17:50:58,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,345 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,345 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=354, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,345 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,345 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,345 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658333) roll requested 2024-11-19T17:50:58,345 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658345 2024-11-19T17:50:58,351 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,351 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,351 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,351 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,351 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,351 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658333 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658345 2024-11-19T17:50:58,353 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,353 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,353 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=355, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,353 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,353 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,353 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658345) roll requested 2024-11-19T17:50:58,353 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658353 2024-11-19T17:50:58,359 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,359 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,359 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,359 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,359 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,359 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658345 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658353 2024-11-19T17:50:58,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,360 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,360 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=356, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,360 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,360 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,360 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658353) roll requested 2024-11-19T17:50:58,360 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658360 2024-11-19T17:50:58,365 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,365 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,365 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,365 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658353 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658360 2024-11-19T17:50:58,367 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,368 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,368 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=357, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,368 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658360) roll requested 2024-11-19T17:50:58,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658368 2024-11-19T17:50:58,374 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,375 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,375 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,375 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,375 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,375 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658360 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658368 2024-11-19T17:50:58,375 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,375 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,375 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=358, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,375 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,376 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,376 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658368) roll requested 2024-11-19T17:50:58,376 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658376 2024-11-19T17:50:58,381 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,381 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,381 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,381 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,381 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,381 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658368 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658376 2024-11-19T17:50:58,382 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,382 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,382 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,382 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=359, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,382 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,382 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658376) roll requested 2024-11-19T17:50:58,382 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658382 2024-11-19T17:50:58,387 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,387 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,387 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,387 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,387 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,387 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658376 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658382 2024-11-19T17:50:58,388 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,388 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,388 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,388 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=360, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,388 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,388 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658382) roll requested 2024-11-19T17:50:58,388 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658388 2024-11-19T17:50:58,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658382 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658388 2024-11-19T17:50:58,394 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,394 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,394 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=361, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,394 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,394 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658388) roll requested 2024-11-19T17:50:58,394 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658394 2024-11-19T17:50:58,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658388 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658394 2024-11-19T17:50:58,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,400 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,400 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=362, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,400 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658394) roll requested 2024-11-19T17:50:58,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658400 2024-11-19T17:50:58,404 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,404 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,404 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,405 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658394 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658400 2024-11-19T17:50:58,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,405 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,405 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=363, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,405 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,405 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658400) roll requested 2024-11-19T17:50:58,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658406 2024-11-19T17:50:58,410 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,411 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,411 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658400 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658406 2024-11-19T17:50:58,411 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,411 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,411 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=364, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,411 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,411 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,411 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658406) roll requested 2024-11-19T17:50:58,412 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658412 2024-11-19T17:50:58,416 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,416 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,416 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,416 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,416 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,416 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658406 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658412 2024-11-19T17:50:58,417 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,417 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,417 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=365, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,417 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,417 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,417 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658412) roll requested 2024-11-19T17:50:58,417 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658417 2024-11-19T17:50:58,422 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,422 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658412 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658417 2024-11-19T17:50:58,423 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,423 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,423 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=366, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,423 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,423 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,423 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658417) roll requested 2024-11-19T17:50:58,423 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658423 2024-11-19T17:50:58,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,428 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,428 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,428 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,428 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,428 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658417 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658423 2024-11-19T17:50:58,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,428 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,429 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=367, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,429 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,429 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,429 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658423) roll requested 2024-11-19T17:50:58,429 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658429 2024-11-19T17:50:58,434 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,434 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,434 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,434 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,434 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,434 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658423 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658429 2024-11-19T17:50:58,435 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,435 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,435 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=368, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,435 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,435 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,435 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658429) roll requested 2024-11-19T17:50:58,435 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658435 2024-11-19T17:50:58,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,441 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,441 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658429 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658435 2024-11-19T17:50:58,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,441 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,442 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=369, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,442 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,442 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,442 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658435) roll requested 2024-11-19T17:50:58,442 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658442 2024-11-19T17:50:58,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,447 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,447 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,447 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658435 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658442 2024-11-19T17:50:58,447 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,448 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,448 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=370, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,448 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,448 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,448 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658442) roll requested 2024-11-19T17:50:58,448 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658448 2024-11-19T17:50:58,452 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,452 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,452 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,453 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,453 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,453 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658442 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658448 2024-11-19T17:50:58,453 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,453 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,453 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=371, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,454 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,454 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,454 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658448) roll requested 2024-11-19T17:50:58,454 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658454 2024-11-19T17:50:58,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,459 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,459 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,459 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658448 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658454 2024-11-19T17:50:58,459 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,459 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,459 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=372, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,460 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,460 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658454) roll requested 2024-11-19T17:50:58,460 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658460 2024-11-19T17:50:58,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658454 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658460 2024-11-19T17:50:58,466 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,466 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,466 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=373, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,466 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658460) roll requested 2024-11-19T17:50:58,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658466 2024-11-19T17:50:58,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658460 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658466 2024-11-19T17:50:58,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,472 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,472 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=374, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,472 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,472 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,472 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658466) roll requested 2024-11-19T17:50:58,472 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658472 2024-11-19T17:50:58,476 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,476 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,476 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,476 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,477 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,477 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658466 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658472 2024-11-19T17:50:58,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,477 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,477 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=375, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,477 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,478 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658472) roll requested 2024-11-19T17:50:58,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658478 2024-11-19T17:50:58,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,483 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,483 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658472 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658478 2024-11-19T17:50:58,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,484 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,484 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=376, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,485 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658478) roll requested 2024-11-19T17:50:58,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658485 2024-11-19T17:50:58,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,490 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,490 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658478 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658485 2024-11-19T17:50:58,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,490 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,491 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=377, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,491 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,491 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658485) roll requested 2024-11-19T17:50:58,491 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658491 2024-11-19T17:50:58,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,496 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658485 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658491 2024-11-19T17:50:58,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,497 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,497 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=378, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,497 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658491) roll requested 2024-11-19T17:50:58,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658497 2024-11-19T17:50:58,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,502 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,502 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,502 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658491 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658497 2024-11-19T17:50:58,503 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,503 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,503 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=379, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,503 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658497) roll requested 2024-11-19T17:50:58,503 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658503 2024-11-19T17:50:58,508 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,508 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,508 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,508 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,508 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,508 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658497 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658503 2024-11-19T17:50:58,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,508 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,508 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=380, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,509 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,509 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,509 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658503) roll requested 2024-11-19T17:50:58,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658509 2024-11-19T17:50:58,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658503 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658509 2024-11-19T17:50:58,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,514 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,514 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=381, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,514 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658509) roll requested 2024-11-19T17:50:58,514 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658514 2024-11-19T17:50:58,519 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,519 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,519 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,519 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,519 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,519 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658509 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658514 2024-11-19T17:50:58,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,520 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,520 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=382, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,520 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,520 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658514) roll requested 2024-11-19T17:50:58,520 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658520 2024-11-19T17:50:58,525 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,525 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658514 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658520 2024-11-19T17:50:58,526 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,527 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=383, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,527 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,527 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658520) roll requested 2024-11-19T17:50:58,527 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658527 2024-11-19T17:50:58,532 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,532 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,532 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,532 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,532 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,532 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658520 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658527 2024-11-19T17:50:58,533 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,533 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=384, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,533 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,533 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,533 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658527) roll requested 2024-11-19T17:50:58,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658533 2024-11-19T17:50:58,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,539 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,539 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658527 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658533 2024-11-19T17:50:58,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,540 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,540 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=385, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,540 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,540 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,540 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658533) roll requested 2024-11-19T17:50:58,540 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658540 2024-11-19T17:50:58,545 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,545 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,545 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,545 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658533 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658540 2024-11-19T17:50:58,546 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,546 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=386, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,546 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,546 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,546 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,546 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658540) roll requested 2024-11-19T17:50:58,546 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658546 2024-11-19T17:50:58,550 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,551 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,551 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,551 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658540 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658546 2024-11-19T17:50:58,551 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,551 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,551 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=387, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,551 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,552 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658546) roll requested 2024-11-19T17:50:58,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658552 2024-11-19T17:50:58,556 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,556 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,556 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,556 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,556 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,557 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658546 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658552 2024-11-19T17:50:58,557 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,557 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,557 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=388, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,557 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658552) roll requested 2024-11-19T17:50:58,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658557 2024-11-19T17:50:58,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,562 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,562 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,562 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,562 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658552 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658557 2024-11-19T17:50:58,563 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,563 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,563 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,563 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=389, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,563 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,563 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658557) roll requested 2024-11-19T17:50:58,563 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658563 2024-11-19T17:50:58,568 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,568 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,568 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,568 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,568 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658557 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658563 2024-11-19T17:50:58,569 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,569 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,569 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=390, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,569 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,569 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,569 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658563) roll requested 2024-11-19T17:50:58,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658569 2024-11-19T17:50:58,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,574 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,574 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,574 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,574 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,574 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658563 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658569 2024-11-19T17:50:58,575 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,575 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,575 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=391, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,575 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,575 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,575 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658569) roll requested 2024-11-19T17:50:58,575 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658575 2024-11-19T17:50:58,580 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,580 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,580 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,580 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,580 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,580 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658569 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658575 2024-11-19T17:50:58,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,580 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,581 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=392, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,581 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,581 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658575) roll requested 2024-11-19T17:50:58,581 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658581 2024-11-19T17:50:58,585 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,585 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,585 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,585 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,585 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,586 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658575 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658581 2024-11-19T17:50:58,586 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,586 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,586 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=393, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,586 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,586 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,586 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658581) roll requested 2024-11-19T17:50:58,587 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658586 2024-11-19T17:50:58,591 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,591 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,591 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658581 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658586 2024-11-19T17:50:58,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,592 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,592 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=394, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,592 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658586) roll requested 2024-11-19T17:50:58,592 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658592 2024-11-19T17:50:58,598 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,599 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658586 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658592 2024-11-19T17:50:58,599 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,599 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,599 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=395, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,599 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,599 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658592) roll requested 2024-11-19T17:50:58,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658599 2024-11-19T17:50:58,604 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,604 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,604 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,604 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,604 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,604 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658592 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658599 2024-11-19T17:50:58,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,604 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,605 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=396, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,605 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,605 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,605 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658599) roll requested 2024-11-19T17:50:58,605 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658605 2024-11-19T17:50:58,609 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,609 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,609 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,609 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,609 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,609 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658599 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658605 2024-11-19T17:50:58,609 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,610 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,610 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=397, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,610 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,610 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,610 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658605) roll requested 2024-11-19T17:50:58,610 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658610 2024-11-19T17:50:58,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,614 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,614 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,614 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,614 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,614 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658605 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658610 2024-11-19T17:50:58,615 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,615 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,615 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=398, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,615 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,615 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,615 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658610) roll requested 2024-11-19T17:50:58,616 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658615 2024-11-19T17:50:58,620 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,621 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,621 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,621 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,621 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,621 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658610 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658615 2024-11-19T17:50:58,622 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,622 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,622 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=399, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,622 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,622 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,622 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658615) roll requested 2024-11-19T17:50:58,622 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658622 2024-11-19T17:50:58,628 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,628 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658615 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658622 2024-11-19T17:50:58,629 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,629 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,629 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=400, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,629 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658622) roll requested 2024-11-19T17:50:58,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658629 2024-11-19T17:50:58,635 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,635 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,636 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,636 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,636 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,636 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658622 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658629 2024-11-19T17:50:58,636 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,637 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,637 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=401, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,637 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658629) roll requested 2024-11-19T17:50:58,637 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658637 2024-11-19T17:50:58,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,644 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,644 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,644 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658629 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658637 2024-11-19T17:50:58,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,646 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,646 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,646 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=402, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,646 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,646 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658637) roll requested 2024-11-19T17:50:58,646 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658646 2024-11-19T17:50:58,653 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,654 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,654 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,654 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,654 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658637 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658646 2024-11-19T17:50:58,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,655 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,655 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=403, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,655 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,655 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,655 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658646) roll requested 2024-11-19T17:50:58,655 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658655 2024-11-19T17:50:58,662 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,663 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658646 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658655 2024-11-19T17:50:58,663 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,664 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,664 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=404, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,664 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658655) roll requested 2024-11-19T17:50:58,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658664 2024-11-19T17:50:58,669 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,669 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,670 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,670 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,670 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,670 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658655 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658664 2024-11-19T17:50:58,671 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,671 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,671 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=405, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,671 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,671 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,671 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658664) roll requested 2024-11-19T17:50:58,671 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658671 2024-11-19T17:50:58,678 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,678 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,678 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,678 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,678 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,678 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658664 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658671 2024-11-19T17:50:58,679 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,679 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,679 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=406, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,679 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,679 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,679 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658671) roll requested 2024-11-19T17:50:58,680 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658679 2024-11-19T17:50:58,685 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,685 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,685 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,685 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,686 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658671 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658679 2024-11-19T17:50:58,686 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,687 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,687 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,687 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=407, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,687 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,687 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658679) roll requested 2024-11-19T17:50:58,687 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658687 2024-11-19T17:50:58,693 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,693 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,693 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,693 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,693 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,693 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658679 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658687 2024-11-19T17:50:58,694 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,694 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,694 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=408, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,694 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,694 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,694 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658687) roll requested 2024-11-19T17:50:58,694 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658694 2024-11-19T17:50:58,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,700 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,700 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,700 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,700 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658687 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658694 2024-11-19T17:50:58,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,701 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,701 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=409, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,701 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658694) roll requested 2024-11-19T17:50:58,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658701 2024-11-19T17:50:58,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658694 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658701 2024-11-19T17:50:58,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,708 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,708 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=410, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,709 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658701) roll requested 2024-11-19T17:50:58,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658709 2024-11-19T17:50:58,712 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,713 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,713 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658701 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658709 2024-11-19T17:50:58,713 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,713 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,714 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=411, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,714 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,714 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658709) roll requested 2024-11-19T17:50:58,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658714 2024-11-19T17:50:58,718 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,718 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,719 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,719 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,719 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658709 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658714 2024-11-19T17:50:58,719 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,719 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,719 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,720 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=412, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,720 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,720 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658714) roll requested 2024-11-19T17:50:58,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658720 2024-11-19T17:50:58,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,725 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658714 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658720 2024-11-19T17:50:58,725 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,725 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,725 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=413, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,725 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,726 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658720) roll requested 2024-11-19T17:50:58,726 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658726 2024-11-19T17:50:58,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658720 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658726 2024-11-19T17:50:58,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,732 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,732 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=414, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,732 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,732 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,732 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658726) roll requested 2024-11-19T17:50:58,732 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658732 2024-11-19T17:50:58,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,738 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,738 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,738 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,738 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658726 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658732 2024-11-19T17:50:58,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,739 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,739 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=415, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,739 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,739 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,739 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658732) roll requested 2024-11-19T17:50:58,739 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658739 2024-11-19T17:50:58,745 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,746 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,746 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,746 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,746 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,746 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658732 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658739 2024-11-19T17:50:58,747 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,747 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,747 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=416, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,747 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,747 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,747 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658739) roll requested 2024-11-19T17:50:58,748 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658747 2024-11-19T17:50:58,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658739 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658747 2024-11-19T17:50:58,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,756 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,756 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=417, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,757 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658747) roll requested 2024-11-19T17:50:58,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658757 2024-11-19T17:50:58,764 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,764 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,764 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,764 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,764 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,764 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658747 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658757 2024-11-19T17:50:58,765 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,765 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,765 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=418, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,765 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,765 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,765 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658757) roll requested 2024-11-19T17:50:58,766 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658765 2024-11-19T17:50:58,771 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,771 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,771 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,772 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658757 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658765 2024-11-19T17:50:58,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,772 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,772 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=419, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,772 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658765) roll requested 2024-11-19T17:50:58,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658773 2024-11-19T17:50:58,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,779 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,779 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,779 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658765 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658773 2024-11-19T17:50:58,779 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,780 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,780 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,780 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=420, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,780 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,780 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658773) roll requested 2024-11-19T17:50:58,780 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658780 2024-11-19T17:50:58,786 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,786 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,786 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,786 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,786 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,786 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658773 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658780 2024-11-19T17:50:58,787 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,787 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,787 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,787 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=421, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,788 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658780) roll requested 2024-11-19T17:50:58,788 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658788 2024-11-19T17:50:58,794 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,794 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,794 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658780 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658788 2024-11-19T17:50:58,795 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,795 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,796 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=422, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,796 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658788) roll requested 2024-11-19T17:50:58,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658796 2024-11-19T17:50:58,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,803 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,803 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,803 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,803 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658788 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658796 2024-11-19T17:50:58,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,804 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,804 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=423, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,805 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658796) roll requested 2024-11-19T17:50:58,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658805 2024-11-19T17:50:58,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658796 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658805 2024-11-19T17:50:58,813 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,814 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,814 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=424, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,814 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,814 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,814 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658805) roll requested 2024-11-19T17:50:58,814 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658814 2024-11-19T17:50:58,823 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,823 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,823 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,823 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,823 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,823 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658805 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658814 2024-11-19T17:50:58,824 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,824 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,824 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=425, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,824 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,825 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,825 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658814) roll requested 2024-11-19T17:50:58,825 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658825 2024-11-19T17:50:58,832 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,832 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,832 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,832 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,832 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,833 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658814 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658825 2024-11-19T17:50:58,834 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,834 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,834 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,834 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=426, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,834 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,834 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658825) roll requested 2024-11-19T17:50:58,834 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658834 2024-11-19T17:50:58,841 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,841 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,841 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,841 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,841 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,841 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658825 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658834 2024-11-19T17:50:58,842 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,842 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,842 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,842 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=427, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,842 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,842 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658834) roll requested 2024-11-19T17:50:58,842 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658842 2024-11-19T17:50:58,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,848 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,848 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658834 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658842 2024-11-19T17:50:58,850 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,850 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,850 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=428, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,850 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,850 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,850 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658842) roll requested 2024-11-19T17:50:58,850 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658850 2024-11-19T17:50:58,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658842 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658850 2024-11-19T17:50:58,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,856 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,856 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,856 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=429, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,856 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,856 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658850) roll requested 2024-11-19T17:50:58,857 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658856 2024-11-19T17:50:58,863 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,863 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,863 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658850 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658856 2024-11-19T17:50:58,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,864 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,864 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=430, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,865 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658856) roll requested 2024-11-19T17:50:58,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658865 2024-11-19T17:50:58,871 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,871 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,871 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,871 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,871 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658856 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658865 2024-11-19T17:50:58,872 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,872 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=431, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,872 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,872 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,872 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,872 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658865) roll requested 2024-11-19T17:50:58,873 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658873 2024-11-19T17:50:58,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658865 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658873 2024-11-19T17:50:58,878 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,878 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,878 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=432, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,878 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658873) roll requested 2024-11-19T17:50:58,879 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658878 2024-11-19T17:50:58,883 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,883 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,883 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,883 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,883 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,883 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658873 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658878 2024-11-19T17:50:58,884 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,884 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,884 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=433, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,884 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,885 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,885 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658878) roll requested 2024-11-19T17:50:58,885 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658885 2024-11-19T17:50:58,891 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,891 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,891 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,891 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,891 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,891 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658878 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658885 2024-11-19T17:50:58,892 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,892 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,893 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=434, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,893 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658885) roll requested 2024-11-19T17:50:58,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658893 2024-11-19T17:50:58,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,900 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,901 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,901 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658885 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658893 2024-11-19T17:50:58,902 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,902 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,902 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=435, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,902 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,902 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,902 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658893) roll requested 2024-11-19T17:50:58,902 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658902 2024-11-19T17:50:58,910 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,910 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,910 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,910 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,910 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658893 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658902 2024-11-19T17:50:58,911 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,911 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,911 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,911 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=436, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,912 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,912 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658902) roll requested 2024-11-19T17:50:58,912 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658912 2024-11-19T17:50:58,916 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,916 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,916 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,916 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,917 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,917 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658902 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658912 2024-11-19T17:50:58,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,917 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,917 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=437, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,917 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658912) roll requested 2024-11-19T17:50:58,918 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658917 2024-11-19T17:50:58,921 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,921 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,922 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,922 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,922 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,922 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658912 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658917 2024-11-19T17:50:58,922 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,922 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,922 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=438, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,922 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,923 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,923 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658917) roll requested 2024-11-19T17:50:58,923 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658923 2024-11-19T17:50:58,928 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,928 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,928 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,928 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,928 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,928 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658917 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658923 2024-11-19T17:50:58,928 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,929 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,929 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=439, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,929 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,929 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,929 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658923) roll requested 2024-11-19T17:50:58,929 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658929 2024-11-19T17:50:58,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,934 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,935 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658923 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658929 2024-11-19T17:50:58,935 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,935 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,936 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,936 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=440, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,936 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,936 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658929) roll requested 2024-11-19T17:50:58,936 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658936 2024-11-19T17:50:58,943 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,943 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,943 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,943 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658929 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658936 2024-11-19T17:50:58,944 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,944 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,945 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=441, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,945 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658936) roll requested 2024-11-19T17:50:58,945 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658945 2024-11-19T17:50:58,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,958 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,958 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,958 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,958 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658936 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658945 2024-11-19T17:50:58,959 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,959 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,959 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=442, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,959 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,959 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,959 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658945) roll requested 2024-11-19T17:50:58,959 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658959 2024-11-19T17:50:58,963 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,963 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,963 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,963 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,963 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,964 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658945 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658959 2024-11-19T17:50:58,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,964 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,964 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=443, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,965 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658959) roll requested 2024-11-19T17:50:58,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658965 2024-11-19T17:50:58,969 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,969 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,969 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,969 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,969 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,969 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658959 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658965 2024-11-19T17:50:58,970 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,970 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,970 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=444, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,970 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,970 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,970 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658965) roll requested 2024-11-19T17:50:58,971 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658970 2024-11-19T17:50:58,975 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,975 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,975 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,975 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,975 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,975 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658965 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658970 2024-11-19T17:50:58,976 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,976 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,976 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,976 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=445, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,976 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,976 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658970) roll requested 2024-11-19T17:50:58,976 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658976 2024-11-19T17:50:58,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,982 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,982 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,982 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658970 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658976 2024-11-19T17:50:58,983 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,983 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,983 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,983 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=446, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,983 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,983 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658976) roll requested 2024-11-19T17:50:58,983 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658983 2024-11-19T17:50:58,989 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,989 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,989 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,989 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,989 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,989 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658976 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658983 2024-11-19T17:50:58,990 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,990 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,990 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=447, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,990 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,991 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658983) roll requested 2024-11-19T17:50:58,991 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658991 2024-11-19T17:50:58,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,997 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,997 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,997 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,997 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:58,997 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658983 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658991 2024-11-19T17:50:58,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:58,998 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,998 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=448, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:58,998 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:58,998 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:58,998 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658991) roll requested 2024-11-19T17:50:58,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038658998 2024-11-19T17:50:59,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,005 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,005 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658991 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658998 2024-11-19T17:50:59,005 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,006 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,006 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=449, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,006 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,006 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,006 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038658998) roll requested 2024-11-19T17:50:59,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659006 2024-11-19T17:50:59,011 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,012 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,012 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,012 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,012 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,012 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038658998 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659006 2024-11-19T17:50:59,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,012 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,013 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,013 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=450, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,013 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,013 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659006) roll requested 2024-11-19T17:50:59,013 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659013 2024-11-19T17:50:59,020 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,020 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,020 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,020 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,021 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,021 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659006 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659013 2024-11-19T17:50:59,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,021 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,021 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=451, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,022 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,022 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,022 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659013) roll requested 2024-11-19T17:50:59,022 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659022 2024-11-19T17:50:59,028 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,028 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,028 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,028 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,028 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659013 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659022 2024-11-19T17:50:59,029 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,029 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,029 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=452, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,030 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,030 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,030 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659022) roll requested 2024-11-19T17:50:59,030 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659030 2024-11-19T17:50:59,036 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,036 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,036 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,036 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,036 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659022 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659030 2024-11-19T17:50:59,037 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,037 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,037 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=453, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,038 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,038 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659030) roll requested 2024-11-19T17:50:59,038 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659038 2024-11-19T17:50:59,045 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,045 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,046 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659030 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659038 2024-11-19T17:50:59,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,047 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,047 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=454, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,047 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,047 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659038) roll requested 2024-11-19T17:50:59,047 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659047 2024-11-19T17:50:59,054 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,054 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,054 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,054 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,055 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,055 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659038 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659047 2024-11-19T17:50:59,055 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,055 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,056 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=455, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,056 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659047) roll requested 2024-11-19T17:50:59,056 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659056 2024-11-19T17:50:59,064 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,064 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,064 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,064 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,065 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659047 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659056 2024-11-19T17:50:59,065 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,065 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,066 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,066 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=456, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,066 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,066 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659056) roll requested 2024-11-19T17:50:59,066 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659066 2024-11-19T17:50:59,074 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,074 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,074 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,074 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,074 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,075 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659056 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659066 2024-11-19T17:50:59,075 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,075 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,076 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=457, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,075 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,076 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,076 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659066) roll requested 2024-11-19T17:50:59,076 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659076 2024-11-19T17:50:59,083 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,083 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,083 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,083 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,084 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659066 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659076 2024-11-19T17:50:59,084 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,085 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,085 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=458, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,085 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,085 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659076) roll requested 2024-11-19T17:50:59,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659085 2024-11-19T17:50:59,092 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,092 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,092 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,092 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,092 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,092 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659076 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659085 2024-11-19T17:50:59,093 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,093 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,094 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=459, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,094 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,094 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,094 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659085) roll requested 2024-11-19T17:50:59,094 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659094 2024-11-19T17:50:59,100 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,100 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,100 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,101 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,101 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,101 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659085 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659094 2024-11-19T17:50:59,101 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,102 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,102 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=460, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,102 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,102 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,102 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659094) roll requested 2024-11-19T17:50:59,102 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659102 2024-11-19T17:50:59,108 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,108 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,108 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,108 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,108 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,108 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659094 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659102 2024-11-19T17:50:59,108 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,109 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,109 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=461, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,109 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,109 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,109 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659102) roll requested 2024-11-19T17:50:59,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659109 2024-11-19T17:50:59,114 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,114 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,114 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,114 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,114 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,114 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659102 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659109 2024-11-19T17:50:59,115 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,115 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,115 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,115 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=462, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,115 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,115 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659109) roll requested 2024-11-19T17:50:59,115 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659115 2024-11-19T17:50:59,121 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,121 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,121 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,121 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,121 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,121 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659109 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659115 2024-11-19T17:50:59,122 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,122 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,122 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=463, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,122 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,123 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,123 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659115) roll requested 2024-11-19T17:50:59,123 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659123 2024-11-19T17:50:59,129 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,129 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,129 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,129 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,129 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,129 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659115 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659123 2024-11-19T17:50:59,130 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,130 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,130 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=464, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,130 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,130 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,130 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659123) roll requested 2024-11-19T17:50:59,130 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659130 2024-11-19T17:50:59,137 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,137 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,137 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,137 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,137 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,137 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659123 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659130 2024-11-19T17:50:59,138 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,138 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,138 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=465, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,138 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,138 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,138 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659130) roll requested 2024-11-19T17:50:59,138 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659138 2024-11-19T17:50:59,145 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,145 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,145 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,145 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,145 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,145 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659130 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659138 2024-11-19T17:50:59,146 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,146 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,147 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=466, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,147 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,147 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,147 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659138) roll requested 2024-11-19T17:50:59,147 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659147 2024-11-19T17:50:59,153 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,153 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,154 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,154 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,154 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,154 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659138 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659147 2024-11-19T17:50:59,155 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,155 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,155 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=467, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,155 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,155 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,155 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659147) roll requested 2024-11-19T17:50:59,155 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659155 2024-11-19T17:50:59,162 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,162 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,162 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,162 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,162 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,162 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659147 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659155 2024-11-19T17:50:59,163 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,163 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,163 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,163 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=468, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,163 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,163 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659155) roll requested 2024-11-19T17:50:59,163 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659163 2024-11-19T17:50:59,169 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,169 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,169 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,170 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,170 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,170 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659155 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659163 2024-11-19T17:50:59,170 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,170 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,171 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,171 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=469, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,171 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,171 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659163) roll requested 2024-11-19T17:50:59,171 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659171 2024-11-19T17:50:59,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659163 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659171 2024-11-19T17:50:59,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,180 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,180 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=470, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,181 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,181 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659171) roll requested 2024-11-19T17:50:59,181 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659181 2024-11-19T17:50:59,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,189 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,189 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,189 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,189 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,189 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659171 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659181 2024-11-19T17:50:59,190 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,190 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,190 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=471, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,190 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,191 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,191 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659181) roll requested 2024-11-19T17:50:59,191 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659191 2024-11-19T17:50:59,200 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,200 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,200 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,200 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,200 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,200 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659181 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659191 2024-11-19T17:50:59,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,201 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,201 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=472, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,201 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,202 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659191) roll requested 2024-11-19T17:50:59,202 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659202 2024-11-19T17:50:59,208 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,209 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659191 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659202 2024-11-19T17:50:59,211 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,211 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,211 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=473, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,211 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,211 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659202) roll requested 2024-11-19T17:50:59,212 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659211 2024-11-19T17:50:59,220 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,220 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,220 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,220 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,220 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,220 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659202 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659211 2024-11-19T17:50:59,221 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,221 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,221 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,221 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=474, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,221 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,221 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659211) roll requested 2024-11-19T17:50:59,221 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659221 2024-11-19T17:50:59,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,229 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,229 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,229 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,229 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659211 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659221 2024-11-19T17:50:59,230 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,230 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,230 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,230 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=475, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,230 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,230 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659221) roll requested 2024-11-19T17:50:59,230 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659230 2024-11-19T17:50:59,238 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,238 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,238 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,239 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,239 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,239 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659221 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659230 2024-11-19T17:50:59,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,240 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,240 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=476, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,240 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,240 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659230) roll requested 2024-11-19T17:50:59,240 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659240 2024-11-19T17:50:59,247 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,248 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,248 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,248 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,248 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,248 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659230 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659240 2024-11-19T17:50:59,249 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,249 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,249 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,249 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=477, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,249 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,249 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659240) roll requested 2024-11-19T17:50:59,249 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659249 2024-11-19T17:50:59,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659240 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659249 2024-11-19T17:50:59,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,256 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,256 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=478, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,256 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,257 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659249) roll requested 2024-11-19T17:50:59,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659257 2024-11-19T17:50:59,263 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,263 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,263 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,263 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,263 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,264 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659249 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659257 2024-11-19T17:50:59,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,264 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,264 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=479, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,265 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,265 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659257) roll requested 2024-11-19T17:50:59,265 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659265 2024-11-19T17:50:59,271 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,271 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,271 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,271 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659257 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659265 2024-11-19T17:50:59,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,272 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,272 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=480, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,272 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,272 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659265) roll requested 2024-11-19T17:50:59,272 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659272 2024-11-19T17:50:59,277 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659265 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659272 2024-11-19T17:50:59,278 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,279 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,279 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=481, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,279 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,279 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,279 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659272) roll requested 2024-11-19T17:50:59,279 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659279 2024-11-19T17:50:59,285 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,285 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,285 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,285 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,285 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,286 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659272 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659279 2024-11-19T17:50:59,286 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,286 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,287 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=482, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,287 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,287 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659279) roll requested 2024-11-19T17:50:59,287 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659287 2024-11-19T17:50:59,293 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,293 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,294 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,294 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,294 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,294 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659279 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659287 2024-11-19T17:50:59,294 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,295 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,295 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=483, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,295 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,295 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,295 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659287) roll requested 2024-11-19T17:50:59,295 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659295 2024-11-19T17:50:59,303 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,303 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,304 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,304 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,304 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,304 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659287 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659295 2024-11-19T17:50:59,305 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,305 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,305 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=484, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,305 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659295) roll requested 2024-11-19T17:50:59,306 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659306 2024-11-19T17:50:59,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,312 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,312 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,312 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,312 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659295 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659306 2024-11-19T17:50:59,313 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,313 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,313 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=485, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,313 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,313 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659306) roll requested 2024-11-19T17:50:59,313 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659313 2024-11-19T17:50:59,318 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,318 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,318 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,319 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659306 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659313 2024-11-19T17:50:59,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,320 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,320 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=486, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,320 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659313) roll requested 2024-11-19T17:50:59,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659320 2024-11-19T17:50:59,326 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,326 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,326 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,327 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,327 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,327 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659313 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659320 2024-11-19T17:50:59,327 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,327 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,327 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=487, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,328 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,328 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659320) roll requested 2024-11-19T17:50:59,328 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659328 2024-11-19T17:50:59,333 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,334 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,334 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659320 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659328 2024-11-19T17:50:59,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,336 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=488, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,336 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,337 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,337 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659328) roll requested 2024-11-19T17:50:59,337 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659337 2024-11-19T17:50:59,342 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,342 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,342 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,342 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,342 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,342 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659328 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659337 2024-11-19T17:50:59,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,344 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,344 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=489, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,345 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,345 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659337) roll requested 2024-11-19T17:50:59,345 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659345 2024-11-19T17:50:59,349 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,349 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,349 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,349 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,349 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,349 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659337 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659345 2024-11-19T17:50:59,350 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,350 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,350 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,350 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=490, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,350 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659345) roll requested 2024-11-19T17:50:59,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659350 2024-11-19T17:50:59,354 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,355 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,355 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659345 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659350 2024-11-19T17:50:59,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,356 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,356 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=491, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,356 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659350) roll requested 2024-11-19T17:50:59,356 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659356 2024-11-19T17:50:59,361 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,361 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,361 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659350 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659356 2024-11-19T17:50:59,362 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,362 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,362 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,362 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=492, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,363 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659356) roll requested 2024-11-19T17:50:59,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659363 2024-11-19T17:50:59,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659356 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659363 2024-11-19T17:50:59,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,368 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,368 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=493, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,368 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659363) roll requested 2024-11-19T17:50:59,369 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659369 2024-11-19T17:50:59,373 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,373 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,373 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,373 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659363 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659369 2024-11-19T17:50:59,373 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,374 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,374 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=494, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,374 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,374 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,374 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659369) roll requested 2024-11-19T17:50:59,374 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659374 2024-11-19T17:50:59,379 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,379 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,379 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,379 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,380 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,380 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659369 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659374 2024-11-19T17:50:59,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,380 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,380 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=495, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,381 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,381 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659374) roll requested 2024-11-19T17:50:59,381 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659381 2024-11-19T17:50:59,386 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,386 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,386 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,386 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659374 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659381 2024-11-19T17:50:59,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,387 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,387 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=496, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,388 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,388 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659381) roll requested 2024-11-19T17:50:59,388 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659388 2024-11-19T17:50:59,394 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,394 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,395 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,395 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,395 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,395 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659381 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659388 2024-11-19T17:50:59,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,396 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,396 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=497, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,396 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,396 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659388) roll requested 2024-11-19T17:50:59,396 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659396 2024-11-19T17:50:59,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,403 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,403 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,403 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659388 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659396 2024-11-19T17:50:59,404 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,404 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,404 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=498, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,404 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,404 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,404 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659396) roll requested 2024-11-19T17:50:59,405 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659404 2024-11-19T17:50:59,410 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659396 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659404 2024-11-19T17:50:59,411 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,411 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,411 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,411 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=499, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,412 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659404) roll requested 2024-11-19T17:50:59,412 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659412 2024-11-19T17:50:59,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659404 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659412 2024-11-19T17:50:59,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,420 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,420 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=500, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,420 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659412) roll requested 2024-11-19T17:50:59,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659421 2024-11-19T17:50:59,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659412 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659421 2024-11-19T17:50:59,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,428 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,428 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=501, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,428 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,429 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659421) roll requested 2024-11-19T17:50:59,429 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659429 2024-11-19T17:50:59,434 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,434 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,434 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,435 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,435 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,435 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659421 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659429 2024-11-19T17:50:59,436 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,436 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,436 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=502, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,436 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,436 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,436 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659429) roll requested 2024-11-19T17:50:59,436 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659436 2024-11-19T17:50:59,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659429 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659436 2024-11-19T17:50:59,446 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,446 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,446 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=503, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,446 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,446 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,446 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659436) roll requested 2024-11-19T17:50:59,447 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659446 2024-11-19T17:50:59,453 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,454 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,454 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,454 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,454 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,454 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659436 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659446 2024-11-19T17:50:59,455 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,455 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,455 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=504, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,455 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,455 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659446) roll requested 2024-11-19T17:50:59,456 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659456 2024-11-19T17:50:59,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,464 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,464 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659446 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659456 2024-11-19T17:50:59,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,465 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,465 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=505, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,465 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659456) roll requested 2024-11-19T17:50:59,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659465 2024-11-19T17:50:59,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,473 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659456 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659465 2024-11-19T17:50:59,474 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,474 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,474 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=506, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,474 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,474 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659465) roll requested 2024-11-19T17:50:59,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659474 2024-11-19T17:50:59,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,479 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,479 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659465 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659474 2024-11-19T17:50:59,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,480 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,480 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=507, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,480 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,480 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659474) roll requested 2024-11-19T17:50:59,480 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659480 2024-11-19T17:50:59,485 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,485 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,486 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659474 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659480 2024-11-19T17:50:59,487 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,487 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,487 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,487 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=508, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,487 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659480) roll requested 2024-11-19T17:50:59,487 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659487 2024-11-19T17:50:59,494 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,494 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,494 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,494 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,494 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,494 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659480 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659487 2024-11-19T17:50:59,495 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,495 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,496 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=509, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,496 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659487) roll requested 2024-11-19T17:50:59,496 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659496 2024-11-19T17:50:59,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659487 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659496 2024-11-19T17:50:59,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,504 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,504 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=510, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,505 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659496) roll requested 2024-11-19T17:50:59,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659505 2024-11-19T17:50:59,512 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659496 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659505 2024-11-19T17:50:59,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,514 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=511, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,514 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,514 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659505) roll requested 2024-11-19T17:50:59,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659515 2024-11-19T17:50:59,522 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,523 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,523 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659505 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659515 2024-11-19T17:50:59,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,524 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,525 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=512, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,525 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,525 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,525 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659515) roll requested 2024-11-19T17:50:59,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659525 2024-11-19T17:50:59,531 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,531 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,532 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,532 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,532 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659515 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659525 2024-11-19T17:50:59,533 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,533 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,533 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,533 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=513, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,533 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659525) roll requested 2024-11-19T17:50:59,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659533 2024-11-19T17:50:59,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,539 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,539 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659525 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659533 2024-11-19T17:50:59,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,540 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,540 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,540 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=514, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,541 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659533) roll requested 2024-11-19T17:50:59,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659541 2024-11-19T17:50:59,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,548 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,548 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659533 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659541 2024-11-19T17:50:59,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,549 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,549 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=515, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,549 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659541) roll requested 2024-11-19T17:50:59,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659549 2024-11-19T17:50:59,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659541 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659549 2024-11-19T17:50:59,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,556 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,556 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,556 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=516, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,557 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659549) roll requested 2024-11-19T17:50:59,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659557 2024-11-19T17:50:59,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659549 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659557 2024-11-19T17:50:59,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,564 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=517, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,564 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,564 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,564 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659557) roll requested 2024-11-19T17:50:59,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659564 2024-11-19T17:50:59,569 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,569 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,569 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,569 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,569 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,569 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659557 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659564 2024-11-19T17:50:59,570 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,570 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,570 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,570 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=518, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,570 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,570 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659564) roll requested 2024-11-19T17:50:59,570 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659570 2024-11-19T17:50:59,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659564 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659570 2024-11-19T17:50:59,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,576 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,576 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=519, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,576 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659570) roll requested 2024-11-19T17:50:59,576 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659576 2024-11-19T17:50:59,582 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,582 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,582 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,582 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659570 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659576 2024-11-19T17:50:59,583 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,583 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,584 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=520, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,584 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659576) roll requested 2024-11-19T17:50:59,584 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659584 2024-11-19T17:50:59,590 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,590 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,590 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,590 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,591 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659576 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659584 2024-11-19T17:50:59,591 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,592 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,592 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=521, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,592 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659584) roll requested 2024-11-19T17:50:59,592 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659592 2024-11-19T17:50:59,599 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,599 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,599 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,599 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,599 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,599 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659584 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659592 2024-11-19T17:50:59,600 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,600 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,600 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,600 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=522, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,600 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,600 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659592) roll requested 2024-11-19T17:50:59,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659601 2024-11-19T17:50:59,605 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,605 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,605 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,605 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,605 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,605 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659592 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659601 2024-11-19T17:50:59,606 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,606 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,606 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=523, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,606 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,606 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,606 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659601) roll requested 2024-11-19T17:50:59,606 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659606 2024-11-19T17:50:59,611 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,611 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,611 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,611 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,611 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,611 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659601 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659606 2024-11-19T17:50:59,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,612 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,612 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=524, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,612 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659606) roll requested 2024-11-19T17:50:59,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659612 2024-11-19T17:50:59,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,618 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,618 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,618 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,618 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,618 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659606 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659612 2024-11-19T17:50:59,619 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,619 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,619 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=525, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,619 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,619 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,619 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659612) roll requested 2024-11-19T17:50:59,620 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659620 2024-11-19T17:50:59,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,626 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,626 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,626 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,626 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,626 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659612 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659620 2024-11-19T17:50:59,627 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,627 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,627 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=526, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,627 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,627 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659620) roll requested 2024-11-19T17:50:59,628 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659628 2024-11-19T17:50:59,632 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,633 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,633 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659620 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659628 2024-11-19T17:50:59,633 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,633 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,633 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=527, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,633 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,634 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,634 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659628) roll requested 2024-11-19T17:50:59,634 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659634 2024-11-19T17:50:59,638 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,638 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,638 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659628 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659634 2024-11-19T17:50:59,639 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,639 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,639 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=528, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,639 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,639 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,639 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659634) roll requested 2024-11-19T17:50:59,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659639 2024-11-19T17:50:59,645 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,645 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,645 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,645 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,646 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659634 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659639 2024-11-19T17:50:59,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,646 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,646 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,646 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=529, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,647 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,647 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659639) roll requested 2024-11-19T17:50:59,647 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659647 2024-11-19T17:50:59,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659639 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659647 2024-11-19T17:50:59,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,654 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,654 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,654 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=530, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,654 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659647) roll requested 2024-11-19T17:50:59,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659654 2024-11-19T17:50:59,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,662 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659647 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659654 2024-11-19T17:50:59,662 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,662 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,662 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,663 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=531, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,663 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,663 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659654) roll requested 2024-11-19T17:50:59,663 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659663 2024-11-19T17:50:59,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,671 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,671 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,671 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,671 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659654 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659663 2024-11-19T17:50:59,673 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,673 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,673 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,673 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=532, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,674 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659663) roll requested 2024-11-19T17:50:59,674 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659674 2024-11-19T17:50:59,681 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,681 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,681 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,681 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,681 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,681 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659663 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659674 2024-11-19T17:50:59,682 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,682 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,682 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,682 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=533, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,682 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,682 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659674) roll requested 2024-11-19T17:50:59,682 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659682 2024-11-19T17:50:59,687 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659674 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659682 2024-11-19T17:50:59,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,689 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,689 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=534, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,689 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659682) roll requested 2024-11-19T17:50:59,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659689 2024-11-19T17:50:59,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659682 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659689 2024-11-19T17:50:59,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,697 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,697 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=535, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,697 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659689) roll requested 2024-11-19T17:50:59,698 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659698 2024-11-19T17:50:59,705 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,705 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,705 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,705 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,705 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,705 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659689 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659698 2024-11-19T17:50:59,706 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,706 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,706 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,707 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=536, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,707 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,707 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659698) roll requested 2024-11-19T17:50:59,707 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659707 2024-11-19T17:50:59,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,713 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,713 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659698 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659707 2024-11-19T17:50:59,714 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,714 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,714 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,715 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=537, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,715 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,715 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659707) roll requested 2024-11-19T17:50:59,715 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659715 2024-11-19T17:50:59,721 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,721 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,721 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,722 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,722 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,722 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659707 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659715 2024-11-19T17:50:59,722 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,723 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,723 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,723 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=538, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,723 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,723 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659715) roll requested 2024-11-19T17:50:59,723 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659723 2024-11-19T17:50:59,730 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,730 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,730 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,730 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,730 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659715 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659723 2024-11-19T17:50:59,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,733 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,733 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=539, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,734 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,734 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659723) roll requested 2024-11-19T17:50:59,734 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659734 2024-11-19T17:50:59,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,738 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659723 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659734 2024-11-19T17:50:59,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,739 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,740 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,740 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=540, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,740 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,740 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659734) roll requested 2024-11-19T17:50:59,740 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659740 2024-11-19T17:50:59,745 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,745 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,745 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659734 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659740 2024-11-19T17:50:59,746 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,746 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,746 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,746 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=541, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,746 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659740) roll requested 2024-11-19T17:50:59,747 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659746 2024-11-19T17:50:59,752 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,752 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,752 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659740 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659746 2024-11-19T17:50:59,753 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,753 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,753 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=542, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,753 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659746) roll requested 2024-11-19T17:50:59,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659753 2024-11-19T17:50:59,759 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659746 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659753 2024-11-19T17:50:59,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,760 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,760 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=543, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,760 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659753) roll requested 2024-11-19T17:50:59,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659760 2024-11-19T17:50:59,767 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659753 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659760 2024-11-19T17:50:59,767 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,767 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,768 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=544, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,768 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659760) roll requested 2024-11-19T17:50:59,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659768 2024-11-19T17:50:59,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,772 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,772 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,772 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659760 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659768 2024-11-19T17:50:59,773 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,773 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,773 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=545, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,773 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659768) roll requested 2024-11-19T17:50:59,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659773 2024-11-19T17:50:59,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,779 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659768 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659773 2024-11-19T17:50:59,779 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,779 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,779 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,779 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=546, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,779 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659773) roll requested 2024-11-19T17:50:59,780 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659779 2024-11-19T17:50:59,786 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,786 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,786 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,786 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,786 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,786 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659773 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659779 2024-11-19T17:50:59,787 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,787 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,787 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=547, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,787 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,787 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,787 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659779) roll requested 2024-11-19T17:50:59,788 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659787 2024-11-19T17:50:59,794 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,794 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,794 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,795 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,795 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659779 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659787 2024-11-19T17:50:59,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,796 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,796 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=548, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,796 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659787) roll requested 2024-11-19T17:50:59,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659796 2024-11-19T17:50:59,802 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,803 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,803 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,803 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,803 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659787 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659796 2024-11-19T17:50:59,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,804 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,804 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=549, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,804 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659796) roll requested 2024-11-19T17:50:59,804 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659804 2024-11-19T17:50:59,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,813 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659796 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659804 2024-11-19T17:50:59,814 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,814 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,814 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,814 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=550, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,815 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659804) roll requested 2024-11-19T17:50:59,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659815 2024-11-19T17:50:59,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,820 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,820 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659804 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659815 2024-11-19T17:50:59,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,821 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,821 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=551, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,822 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,822 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659815) roll requested 2024-11-19T17:50:59,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659822 2024-11-19T17:50:59,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659815 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659822 2024-11-19T17:50:59,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,828 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,828 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=552, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,829 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659822) roll requested 2024-11-19T17:50:59,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659829 2024-11-19T17:50:59,834 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,834 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,835 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,835 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,835 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,835 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659822 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659829 2024-11-19T17:50:59,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,836 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,836 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=553, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,836 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,836 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659829) roll requested 2024-11-19T17:50:59,836 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659836 2024-11-19T17:50:59,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,843 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,843 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,843 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,843 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,844 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659829 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659836 2024-11-19T17:50:59,844 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,844 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,845 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,845 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=554, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,845 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659836) roll requested 2024-11-19T17:50:59,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659845 2024-11-19T17:50:59,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659836 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659845 2024-11-19T17:50:59,854 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,854 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,854 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,854 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=555, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,854 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,854 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659845) roll requested 2024-11-19T17:50:59,855 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659854 2024-11-19T17:50:59,861 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,861 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,861 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,861 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,861 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,861 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659845 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659854 2024-11-19T17:50:59,862 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,862 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,862 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=556, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,862 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,862 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,862 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659854) roll requested 2024-11-19T17:50:59,863 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659863 2024-11-19T17:50:59,869 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,869 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,869 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,869 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,869 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,869 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659854 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659863 2024-11-19T17:50:59,870 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,870 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,870 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,870 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=557, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,870 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,870 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659863) roll requested 2024-11-19T17:50:59,871 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659870 2024-11-19T17:50:59,876 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,876 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659863 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659870 2024-11-19T17:50:59,877 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,878 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,878 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=558, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,878 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659870) roll requested 2024-11-19T17:50:59,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659878 2024-11-19T17:50:59,885 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,885 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,886 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659870 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659878 2024-11-19T17:50:59,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,887 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,887 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=559, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,887 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,887 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659878) roll requested 2024-11-19T17:50:59,887 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659887 2024-11-19T17:50:59,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,894 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,894 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,894 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,894 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,894 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659878 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659887 2024-11-19T17:50:59,895 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,895 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,895 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,895 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=560, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,895 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,895 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659887) roll requested 2024-11-19T17:50:59,895 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659895 2024-11-19T17:50:59,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,900 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,901 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,901 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659887 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659895 2024-11-19T17:50:59,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,901 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,901 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=561, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,902 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,902 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659895) roll requested 2024-11-19T17:50:59,902 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659902 2024-11-19T17:50:59,914 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,914 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,914 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,915 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,915 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,915 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659895 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659902 2024-11-19T17:50:59,915 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,915 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,916 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=562, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,916 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,916 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,916 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659902) roll requested 2024-11-19T17:50:59,916 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659916 2024-11-19T17:50:59,920 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659902 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659916 2024-11-19T17:50:59,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,921 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,921 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=563, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,921 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659916) roll requested 2024-11-19T17:50:59,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659921 2024-11-19T17:50:59,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,926 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659916 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659921 2024-11-19T17:50:59,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,927 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,927 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,927 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=564, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,928 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659921) roll requested 2024-11-19T17:50:59,928 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659928 2024-11-19T17:50:59,933 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,933 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,933 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,933 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659921 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659928 2024-11-19T17:50:59,934 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,934 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,934 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,934 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=565, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,935 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,935 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659928) roll requested 2024-11-19T17:50:59,935 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659935 2024-11-19T17:50:59,941 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,941 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,941 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,941 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,941 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,941 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659928 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659935 2024-11-19T17:50:59,942 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,942 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,942 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,943 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=566, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,943 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,943 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659935) roll requested 2024-11-19T17:50:59,943 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659943 2024-11-19T17:50:59,950 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,950 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,950 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,950 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,950 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659935 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659943 2024-11-19T17:50:59,951 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,951 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,951 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,952 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=567, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,952 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659943) roll requested 2024-11-19T17:50:59,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659952 2024-11-19T17:50:59,959 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,959 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,959 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659943 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659952 2024-11-19T17:50:59,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,960 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,960 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=568, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,960 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659952) roll requested 2024-11-19T17:50:59,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659960 2024-11-19T17:50:59,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,966 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,966 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659952 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659960 2024-11-19T17:50:59,967 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,967 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,967 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,967 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=569, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,967 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,967 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659960) roll requested 2024-11-19T17:50:59,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659968 2024-11-19T17:50:59,974 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,974 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,974 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,975 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,975 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,975 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659960 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659968 2024-11-19T17:50:59,976 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,976 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,976 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,976 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=570, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,976 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,976 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659968) roll requested 2024-11-19T17:50:59,976 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659976 2024-11-19T17:50:59,983 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,983 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,983 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,983 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,984 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659968 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659976 2024-11-19T17:50:59,985 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,985 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,985 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=571, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,986 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,986 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659976) roll requested 2024-11-19T17:50:59,986 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659986 2024-11-19T17:50:59,994 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,994 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,994 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,994 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,994 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:50:59,994 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659976 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659986 2024-11-19T17:50:59,995 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:50:59,995 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,995 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:50:59,996 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=572, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:50:59,996 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:50:59,996 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659986) roll requested 2024-11-19T17:50:59,996 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038659996 2024-11-19T17:51:00,003 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659986 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659996 2024-11-19T17:51:00,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,004 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,004 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=573, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,004 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038659996) roll requested 2024-11-19T17:51:00,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660004 2024-11-19T17:51:00,009 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,009 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,009 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038659996 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660004 2024-11-19T17:51:00,009 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,009 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,010 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,010 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=574, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,010 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,010 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660004) roll requested 2024-11-19T17:51:00,010 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660010 2024-11-19T17:51:00,014 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,014 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,015 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,015 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,015 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660004 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660010 2024-11-19T17:51:00,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,016 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,016 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=575, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,016 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,016 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660010) roll requested 2024-11-19T17:51:00,016 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660016 2024-11-19T17:51:00,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660010 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660016 2024-11-19T17:51:00,023 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,023 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,023 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,023 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=576, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,024 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,024 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660016) roll requested 2024-11-19T17:51:00,024 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660024 2024-11-19T17:51:00,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660016 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660024 2024-11-19T17:51:00,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,032 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,033 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=577, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,033 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660024) roll requested 2024-11-19T17:51:00,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660033 2024-11-19T17:51:00,039 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,039 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,040 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660024 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660033 2024-11-19T17:51:00,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,041 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,041 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=578, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,041 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660033) roll requested 2024-11-19T17:51:00,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660041 2024-11-19T17:51:00,048 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,048 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,048 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,048 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,048 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,048 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660033 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660041 2024-11-19T17:51:00,049 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,049 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,049 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=579, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,049 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,049 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660041) roll requested 2024-11-19T17:51:00,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660049 2024-11-19T17:51:00,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,057 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,057 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,057 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,057 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,057 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660041 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660049 2024-11-19T17:51:00,058 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,058 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,058 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,058 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=580, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,058 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,058 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660049) roll requested 2024-11-19T17:51:00,059 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660058 2024-11-19T17:51:00,066 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,066 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,066 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,066 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,066 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,066 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660049 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660058 2024-11-19T17:51:00,067 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,067 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,067 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,067 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=581, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,068 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,068 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660058) roll requested 2024-11-19T17:51:00,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660068 2024-11-19T17:51:00,075 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,075 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660058 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660068 2024-11-19T17:51:00,076 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,076 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,076 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,077 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=582, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,077 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660068) roll requested 2024-11-19T17:51:00,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660077 2024-11-19T17:51:00,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660068 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660077 2024-11-19T17:51:00,085 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,085 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,086 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,086 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=583, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,086 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,086 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660077) roll requested 2024-11-19T17:51:00,086 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660086 2024-11-19T17:51:00,094 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,094 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660077 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660086 2024-11-19T17:51:00,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,096 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,096 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=584, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,097 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660086) roll requested 2024-11-19T17:51:00,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660097 2024-11-19T17:51:00,102 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660086 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660097 2024-11-19T17:51:00,103 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,103 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,103 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,104 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=585, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,104 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,104 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660097) roll requested 2024-11-19T17:51:00,104 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660104 2024-11-19T17:51:00,108 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,108 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,109 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,109 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,109 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660097 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660104 2024-11-19T17:51:00,109 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,109 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,110 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,110 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=586, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,110 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,110 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660104) roll requested 2024-11-19T17:51:00,110 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660110 2024-11-19T17:51:00,116 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,116 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,116 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,116 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,116 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,116 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660104 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660110 2024-11-19T17:51:00,117 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,117 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,117 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,117 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=587, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,117 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,117 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660110) roll requested 2024-11-19T17:51:00,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660117 2024-11-19T17:51:00,124 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,124 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,124 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,124 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,124 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,124 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660110 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660117 2024-11-19T17:51:00,125 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,125 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,125 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,126 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=588, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,126 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,126 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660117) roll requested 2024-11-19T17:51:00,126 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660126 2024-11-19T17:51:00,134 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,134 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,134 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,135 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,135 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660117 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660126 2024-11-19T17:51:00,135 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,136 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,136 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,136 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=589, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,136 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,136 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660126) roll requested 2024-11-19T17:51:00,136 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660136 2024-11-19T17:51:00,143 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,143 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,144 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,144 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,144 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,144 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660126 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660136 2024-11-19T17:51:00,144 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,144 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,145 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=590, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,145 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660136) roll requested 2024-11-19T17:51:00,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660145 2024-11-19T17:51:00,150 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,150 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,151 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660136 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660145 2024-11-19T17:51:00,151 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,151 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,152 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=591, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,152 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,152 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660145) roll requested 2024-11-19T17:51:00,152 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660152 2024-11-19T17:51:00,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660145 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660152 2024-11-19T17:51:00,158 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,159 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,159 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=592, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,159 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,159 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660152) roll requested 2024-11-19T17:51:00,159 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660159 2024-11-19T17:51:00,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,165 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,165 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,165 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660152 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660159 2024-11-19T17:51:00,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,165 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,166 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,166 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=593, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,166 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,166 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660159) roll requested 2024-11-19T17:51:00,166 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660166 2024-11-19T17:51:00,173 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,173 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,173 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,173 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,173 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,173 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660159 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660166 2024-11-19T17:51:00,173 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,173 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,174 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=594, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,174 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,174 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,174 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660166) roll requested 2024-11-19T17:51:00,174 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660174 2024-11-19T17:51:00,180 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,180 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,180 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,180 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,180 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,180 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660166 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660174 2024-11-19T17:51:00,181 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,181 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,182 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,182 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=595, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,182 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,182 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660174) roll requested 2024-11-19T17:51:00,182 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660182 2024-11-19T17:51:00,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,188 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660174 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660182 2024-11-19T17:51:00,192 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,192 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,192 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,192 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=596, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,193 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,193 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660182) roll requested 2024-11-19T17:51:00,193 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660193 2024-11-19T17:51:00,199 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,199 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,200 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,200 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,200 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,200 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660182 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660193 2024-11-19T17:51:00,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,201 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,201 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=597, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,201 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660193) roll requested 2024-11-19T17:51:00,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660201 2024-11-19T17:51:00,207 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,207 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,207 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,207 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,207 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660193 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660201 2024-11-19T17:51:00,207 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,208 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,208 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=598, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,208 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,208 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660201) roll requested 2024-11-19T17:51:00,208 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660208 2024-11-19T17:51:00,213 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,213 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,213 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,214 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660201 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660208 2024-11-19T17:51:00,214 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,214 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,214 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,214 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=599, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,215 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,215 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660208) roll requested 2024-11-19T17:51:00,215 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660215 2024-11-19T17:51:00,219 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,219 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,219 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,219 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660208 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660215 2024-11-19T17:51:00,220 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,220 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,220 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,220 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=600, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,220 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,220 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660215) roll requested 2024-11-19T17:51:00,221 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660220 2024-11-19T17:51:00,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,229 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660215 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660220 2024-11-19T17:51:00,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,232 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,232 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=601, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,232 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,232 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660220) roll requested 2024-11-19T17:51:00,232 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660232 2024-11-19T17:51:00,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660220 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660232 2024-11-19T17:51:00,237 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,237 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,237 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,238 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=602, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,238 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660232) roll requested 2024-11-19T17:51:00,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660238 2024-11-19T17:51:00,242 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,242 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,242 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,242 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,242 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,242 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660232 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660238 2024-11-19T17:51:00,242 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,243 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,243 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,243 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=603, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,243 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,243 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660238) roll requested 2024-11-19T17:51:00,243 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660243 2024-11-19T17:51:00,248 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,248 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,248 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,249 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,249 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,249 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660238 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660243 2024-11-19T17:51:00,249 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,249 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,249 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,250 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=604, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,250 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,250 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660243) roll requested 2024-11-19T17:51:00,250 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660250 2024-11-19T17:51:00,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,255 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660243 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660250 2024-11-19T17:51:00,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,256 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,256 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,256 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=605, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,256 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,256 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660250) roll requested 2024-11-19T17:51:00,256 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660256 2024-11-19T17:51:00,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,263 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660250 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660256 2024-11-19T17:51:00,263 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,263 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,263 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,264 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=606, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,264 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660256) roll requested 2024-11-19T17:51:00,264 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660264 2024-11-19T17:51:00,270 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,270 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,270 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,270 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,270 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,270 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660256 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660264 2024-11-19T17:51:00,271 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,271 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,271 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,271 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=607, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,271 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,271 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660264) roll requested 2024-11-19T17:51:00,271 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660271 2024-11-19T17:51:00,277 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,277 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,277 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660264 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660271 2024-11-19T17:51:00,278 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,278 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,278 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,278 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=608, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,278 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,278 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660271) roll requested 2024-11-19T17:51:00,279 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660278 2024-11-19T17:51:00,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,286 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660271 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660278 2024-11-19T17:51:00,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,288 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,288 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,288 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=609, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,288 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660278) roll requested 2024-11-19T17:51:00,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660288 2024-11-19T17:51:00,295 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,295 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,295 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,295 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660278 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660288 2024-11-19T17:51:00,296 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,296 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,296 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,296 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=610, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,296 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,296 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660288) roll requested 2024-11-19T17:51:00,296 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660296 2024-11-19T17:51:00,302 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,302 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,302 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,302 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,302 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,302 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660288 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660296 2024-11-19T17:51:00,303 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,303 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,303 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,303 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=611, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,303 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,303 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660296) roll requested 2024-11-19T17:51:00,303 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660303 2024-11-19T17:51:00,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660296 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660303 2024-11-19T17:51:00,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,312 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,312 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=612, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,312 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660303) roll requested 2024-11-19T17:51:00,313 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660313 2024-11-19T17:51:00,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,320 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,320 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,320 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,320 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660303 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660313 2024-11-19T17:51:00,321 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,321 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,321 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,321 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=613, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,321 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,321 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660313) roll requested 2024-11-19T17:51:00,322 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660322 2024-11-19T17:51:00,329 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,329 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,330 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660313 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660322 2024-11-19T17:51:00,331 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,331 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,331 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,331 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=614, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,332 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,332 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660322) roll requested 2024-11-19T17:51:00,332 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660332 2024-11-19T17:51:00,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,339 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,340 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,340 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660322 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660332 2024-11-19T17:51:00,341 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,341 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,341 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,341 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=615, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,342 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,342 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660332) roll requested 2024-11-19T17:51:00,342 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660342 2024-11-19T17:51:00,349 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,349 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,349 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,349 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,350 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,350 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660332 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660342 2024-11-19T17:51:00,351 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,351 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,351 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=616, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,351 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,351 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,351 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660342) roll requested 2024-11-19T17:51:00,351 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660351 2024-11-19T17:51:00,358 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,358 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,358 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,358 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,358 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,358 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660342 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660351 2024-11-19T17:51:00,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,360 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,360 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,360 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=617, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,360 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,360 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660351) roll requested 2024-11-19T17:51:00,360 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660360 2024-11-19T17:51:00,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660351 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660360 2024-11-19T17:51:00,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,369 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,369 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=618, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,369 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660360) roll requested 2024-11-19T17:51:00,369 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660369 2024-11-19T17:51:00,377 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,377 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660360 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660369 2024-11-19T17:51:00,378 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,378 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,378 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,378 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=619, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,378 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,378 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660369) roll requested 2024-11-19T17:51:00,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660378 2024-11-19T17:51:00,384 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,384 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,384 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,384 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,385 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660369 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660378 2024-11-19T17:51:00,385 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,385 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,385 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,385 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=620, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,385 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,386 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660378) roll requested 2024-11-19T17:51:00,386 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660386 2024-11-19T17:51:00,392 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,392 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,392 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,392 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,392 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660378 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660386 2024-11-19T17:51:00,393 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,393 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,393 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,393 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=621, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,394 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660386) roll requested 2024-11-19T17:51:00,394 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660394 2024-11-19T17:51:00,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,402 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,402 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,402 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,402 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660386 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660394 2024-11-19T17:51:00,403 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,403 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,403 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,403 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=622, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,404 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,404 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660394) roll requested 2024-11-19T17:51:00,404 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660404 2024-11-19T17:51:00,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,411 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,411 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,412 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,412 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,412 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660394 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660404 2024-11-19T17:51:00,412 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,412 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,413 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=623, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,413 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660404) roll requested 2024-11-19T17:51:00,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660413 2024-11-19T17:51:00,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660404 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660413 2024-11-19T17:51:00,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,420 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,420 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=624, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,420 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660413) roll requested 2024-11-19T17:51:00,420 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660420 2024-11-19T17:51:00,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,426 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,426 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660413 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660420 2024-11-19T17:51:00,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,427 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,427 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=625, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,427 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660420) roll requested 2024-11-19T17:51:00,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660428 2024-11-19T17:51:00,434 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,434 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,434 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,434 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,434 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,435 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660420 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660428 2024-11-19T17:51:00,435 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,436 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,436 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,436 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=626, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,436 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,436 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660428) roll requested 2024-11-19T17:51:00,436 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660436 2024-11-19T17:51:00,443 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,443 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,443 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,443 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,443 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,443 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660428 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660436 2024-11-19T17:51:00,444 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,444 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,444 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,444 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=627, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,444 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,444 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660436) roll requested 2024-11-19T17:51:00,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660444 2024-11-19T17:51:00,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,451 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,451 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,451 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,451 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660436 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660444 2024-11-19T17:51:00,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,452 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,453 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=628, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,453 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,453 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660444) roll requested 2024-11-19T17:51:00,453 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660453 2024-11-19T17:51:00,460 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,460 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,460 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,460 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,461 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660444 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660453 2024-11-19T17:51:00,462 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,462 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,462 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,462 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=629, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,462 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,462 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660453) roll requested 2024-11-19T17:51:00,462 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660462 2024-11-19T17:51:00,469 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,469 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,469 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,469 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,469 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,469 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660453 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660462 2024-11-19T17:51:00,470 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,470 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,470 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,471 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=630, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,471 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,471 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660462) roll requested 2024-11-19T17:51:00,471 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660471 2024-11-19T17:51:00,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,479 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660462 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660471 2024-11-19T17:51:00,481 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,481 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,481 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,481 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=631, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,481 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,482 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660471) roll requested 2024-11-19T17:51:00,482 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660482 2024-11-19T17:51:00,488 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,488 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,488 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,488 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,488 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,488 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660471 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660482 2024-11-19T17:51:00,488 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,489 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,489 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,489 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=632, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,489 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,489 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660482) roll requested 2024-11-19T17:51:00,489 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660489 2024-11-19T17:51:00,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,496 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660482 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660489 2024-11-19T17:51:00,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,497 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,497 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=633, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,497 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660489) roll requested 2024-11-19T17:51:00,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660497 2024-11-19T17:51:00,504 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,504 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,504 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,504 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,504 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,504 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660489 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660497 2024-11-19T17:51:00,505 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,505 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,505 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,505 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=634, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,505 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,506 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660497) roll requested 2024-11-19T17:51:00,506 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660506 2024-11-19T17:51:00,512 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,512 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,512 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,512 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,512 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,512 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660497 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660506 2024-11-19T17:51:00,513 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,513 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,513 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,514 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=635, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,514 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660506) roll requested 2024-11-19T17:51:00,514 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660514 2024-11-19T17:51:00,522 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,522 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,522 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,522 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,522 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,522 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660506 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660514 2024-11-19T17:51:00,523 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,523 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,523 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,524 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=636, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,524 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,524 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660514) roll requested 2024-11-19T17:51:00,524 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660524 2024-11-19T17:51:00,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,530 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,530 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,530 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660514 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660524 2024-11-19T17:51:00,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,531 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,532 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,532 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=637, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,532 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,532 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660524) roll requested 2024-11-19T17:51:00,532 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660532 2024-11-19T17:51:00,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,538 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660524 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660532 2024-11-19T17:51:00,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,539 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,539 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,539 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=638, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,539 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,539 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660532) roll requested 2024-11-19T17:51:00,539 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660539 2024-11-19T17:51:00,545 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,545 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,546 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660532 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660539 2024-11-19T17:51:00,546 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,547 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,547 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,547 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=639, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,547 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,547 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660539) roll requested 2024-11-19T17:51:00,547 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660547 2024-11-19T17:51:00,556 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,556 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,556 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,557 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,557 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,557 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660539 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660547 2024-11-19T17:51:00,557 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,557 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,557 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=640, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,558 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,558 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660547) roll requested 2024-11-19T17:51:00,558 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660558 2024-11-19T17:51:00,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660547 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660558 2024-11-19T17:51:00,563 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,564 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,564 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=641, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,564 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,564 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660558) roll requested 2024-11-19T17:51:00,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660564 2024-11-19T17:51:00,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660558 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660564 2024-11-19T17:51:00,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,571 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,572 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=642, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,572 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660564) roll requested 2024-11-19T17:51:00,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660572 2024-11-19T17:51:00,576 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,576 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,576 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,577 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,577 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660564 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660572 2024-11-19T17:51:00,577 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,577 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,577 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=643, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,577 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660572) roll requested 2024-11-19T17:51:00,578 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660578 2024-11-19T17:51:00,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660572 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660578 2024-11-19T17:51:00,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,584 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,584 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=644, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,584 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660578) roll requested 2024-11-19T17:51:00,584 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660584 2024-11-19T17:51:00,589 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,589 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,589 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,590 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,590 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,590 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660578 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660584 2024-11-19T17:51:00,590 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,590 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,590 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,590 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=645, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,591 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,591 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660584) roll requested 2024-11-19T17:51:00,591 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660591 2024-11-19T17:51:00,598 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,599 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,599 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660584 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660591 2024-11-19T17:51:00,600 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,600 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,600 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,600 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=646, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,600 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,600 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660591) roll requested 2024-11-19T17:51:00,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660600 2024-11-19T17:51:00,608 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,608 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,609 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,609 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660591 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660600 2024-11-19T17:51:00,609 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,609 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,609 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,609 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=647, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,609 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660600) roll requested 2024-11-19T17:51:00,610 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660609 2024-11-19T17:51:00,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,614 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,614 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,614 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,614 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,614 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660600 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660609 2024-11-19T17:51:00,615 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,615 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,615 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,615 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=648, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,615 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,615 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660609) roll requested 2024-11-19T17:51:00,615 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660615 2024-11-19T17:51:00,620 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,620 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,620 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,620 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,621 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,621 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660609 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660615 2024-11-19T17:51:00,621 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,621 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,622 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,622 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=649, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,622 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,622 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660615) roll requested 2024-11-19T17:51:00,622 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660622 2024-11-19T17:51:00,627 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,627 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,627 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660615 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660622 2024-11-19T17:51:00,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,628 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,628 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=650, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,628 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660622) roll requested 2024-11-19T17:51:00,628 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660628 2024-11-19T17:51:00,633 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,633 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,633 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660622 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660628 2024-11-19T17:51:00,634 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,634 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,634 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,634 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=651, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,634 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,634 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660628) roll requested 2024-11-19T17:51:00,634 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660634 2024-11-19T17:51:00,639 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,639 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,639 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,639 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,639 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,639 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660628 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660634 2024-11-19T17:51:00,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,640 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,640 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,640 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=652, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,640 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,640 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660634) roll requested 2024-11-19T17:51:00,640 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660640 2024-11-19T17:51:00,645 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,645 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,645 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,645 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,646 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660634 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660640 2024-11-19T17:51:00,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,646 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,646 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,646 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=653, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,646 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,646 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660640) roll requested 2024-11-19T17:51:00,646 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660646 2024-11-19T17:51:00,651 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,651 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,651 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660640 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660646 2024-11-19T17:51:00,651 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,651 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,651 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,651 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=654, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,652 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,652 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660646) roll requested 2024-11-19T17:51:00,652 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660652 2024-11-19T17:51:00,656 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,656 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,656 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,656 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,656 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,656 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660646 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660652 2024-11-19T17:51:00,657 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,657 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,657 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,658 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=655, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,658 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,658 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660652) roll requested 2024-11-19T17:51:00,658 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660658 2024-11-19T17:51:00,662 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,662 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,662 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,662 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,662 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,662 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660652 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660658 2024-11-19T17:51:00,663 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,663 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,663 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,664 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=656, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,664 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660658) roll requested 2024-11-19T17:51:00,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660664 2024-11-19T17:51:00,668 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,668 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,668 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,668 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,668 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,669 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660658 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660664 2024-11-19T17:51:00,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,669 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,669 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=657, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,670 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,670 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660664) roll requested 2024-11-19T17:51:00,670 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660670 2024-11-19T17:51:00,674 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,674 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,674 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,674 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,674 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,674 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660664 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660670 2024-11-19T17:51:00,675 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,675 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,675 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,675 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=658, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,675 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,675 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660670) roll requested 2024-11-19T17:51:00,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660675 2024-11-19T17:51:00,680 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,680 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,680 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,680 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660670 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660675 2024-11-19T17:51:00,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,680 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,681 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=659, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,681 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660675) roll requested 2024-11-19T17:51:00,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660681 2024-11-19T17:51:00,685 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,685 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,685 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,685 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,685 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660675 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660681 2024-11-19T17:51:00,686 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,686 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,686 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,686 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=660, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,687 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,687 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660681) roll requested 2024-11-19T17:51:00,687 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660687 2024-11-19T17:51:00,691 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,691 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,691 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,691 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,691 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,691 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660681 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660687 2024-11-19T17:51:00,692 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,692 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,692 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,692 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=661, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,692 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,692 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660687) roll requested 2024-11-19T17:51:00,692 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660692 2024-11-19T17:51:00,697 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,697 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,697 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,697 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,697 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,697 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660687 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660692 2024-11-19T17:51:00,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,697 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,698 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=662, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,698 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,698 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660692) roll requested 2024-11-19T17:51:00,698 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660698 2024-11-19T17:51:00,702 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,702 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,702 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660692 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660698 2024-11-19T17:51:00,703 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,703 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,703 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,703 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=663, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,703 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660698) roll requested 2024-11-19T17:51:00,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660704 2024-11-19T17:51:00,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660698 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660704 2024-11-19T17:51:00,709 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,709 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,709 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=664, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,709 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660704) roll requested 2024-11-19T17:51:00,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660709 2024-11-19T17:51:00,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,714 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,714 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660704 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660709 2024-11-19T17:51:00,715 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,715 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,715 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,716 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=665, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,716 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,716 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660709) roll requested 2024-11-19T17:51:00,716 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660716 2024-11-19T17:51:00,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,725 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660709 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660716 2024-11-19T17:51:00,725 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,725 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,725 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,725 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=666, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,726 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660716) roll requested 2024-11-19T17:51:00,726 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660726 2024-11-19T17:51:00,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660716 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660726 2024-11-19T17:51:00,735 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,735 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,735 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,735 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=667, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,735 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,735 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660726) roll requested 2024-11-19T17:51:00,735 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660735 2024-11-19T17:51:00,740 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,740 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,740 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660726 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660735 2024-11-19T17:51:00,741 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,741 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,741 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,741 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=668, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,741 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,741 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660735) roll requested 2024-11-19T17:51:00,742 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660742 2024-11-19T17:51:00,746 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,746 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,746 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,746 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,746 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,746 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660735 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660742 2024-11-19T17:51:00,747 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,747 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,747 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,747 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=669, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,747 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,747 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660742) roll requested 2024-11-19T17:51:00,747 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660747 2024-11-19T17:51:00,752 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,752 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,752 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660742 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660747 2024-11-19T17:51:00,753 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,753 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,753 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=670, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,753 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660747) roll requested 2024-11-19T17:51:00,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660753 2024-11-19T17:51:00,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660747 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660753 2024-11-19T17:51:00,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,759 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,759 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=671, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,759 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660753) roll requested 2024-11-19T17:51:00,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660759 2024-11-19T17:51:00,764 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,764 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,764 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,764 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,765 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,765 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660753 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660759 2024-11-19T17:51:00,765 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,765 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,765 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,765 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=672, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,766 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,766 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660759) roll requested 2024-11-19T17:51:00,766 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660766 2024-11-19T17:51:00,771 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,771 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,771 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,771 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660759 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660766 2024-11-19T17:51:00,771 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,771 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,771 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,771 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=673, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,772 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660766) roll requested 2024-11-19T17:51:00,772 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660772 2024-11-19T17:51:00,776 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,776 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,777 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,777 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660766 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660772 2024-11-19T17:51:00,777 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,777 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,777 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,778 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=674, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,778 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,778 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660772) roll requested 2024-11-19T17:51:00,778 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660778 2024-11-19T17:51:00,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,782 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,782 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,782 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,782 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,783 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660772 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660778 2024-11-19T17:51:00,783 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,783 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,783 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,783 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=675, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,783 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,783 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660778) roll requested 2024-11-19T17:51:00,783 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660783 2024-11-19T17:51:00,788 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,788 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,788 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,788 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,788 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,788 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660778 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660783 2024-11-19T17:51:00,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,789 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,789 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=676, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,789 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,789 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660783) roll requested 2024-11-19T17:51:00,790 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660789 2024-11-19T17:51:00,794 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,794 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,794 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660783 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660789 2024-11-19T17:51:00,795 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,795 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,795 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,795 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=677, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,795 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,795 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660789) roll requested 2024-11-19T17:51:00,795 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660795 2024-11-19T17:51:00,800 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,800 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,800 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,800 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,800 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,800 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660789 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660795 2024-11-19T17:51:00,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,800 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,800 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,801 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=678, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,801 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,801 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660795) roll requested 2024-11-19T17:51:00,801 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660801 2024-11-19T17:51:00,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,806 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,806 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,806 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,806 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,806 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660795 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660801 2024-11-19T17:51:00,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,806 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,806 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=679, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,807 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,807 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660801) roll requested 2024-11-19T17:51:00,807 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660807 2024-11-19T17:51:00,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660801 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660807 2024-11-19T17:51:00,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,812 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,812 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=680, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,812 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660807) roll requested 2024-11-19T17:51:00,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660812 2024-11-19T17:51:00,817 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,817 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,817 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,817 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,817 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,817 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660807 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660812 2024-11-19T17:51:00,818 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,818 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,818 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,818 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=681, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,818 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,818 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660812) roll requested 2024-11-19T17:51:00,818 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660818 2024-11-19T17:51:00,823 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,823 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,823 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,823 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,823 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,823 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660812 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660818 2024-11-19T17:51:00,824 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,824 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,824 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,824 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=682, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,824 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,824 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660818) roll requested 2024-11-19T17:51:00,824 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660824 2024-11-19T17:51:00,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,830 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660818 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660824 2024-11-19T17:51:00,830 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,830 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,830 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,830 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=683, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,830 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,830 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660824) roll requested 2024-11-19T17:51:00,830 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660830 2024-11-19T17:51:00,835 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,835 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,835 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,835 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,835 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,835 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660824 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660830 2024-11-19T17:51:00,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,836 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,836 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=684, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,836 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,836 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660830) roll requested 2024-11-19T17:51:00,836 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660836 2024-11-19T17:51:00,841 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,841 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,841 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,841 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,841 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,841 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660830 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660836 2024-11-19T17:51:00,842 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,842 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,842 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,842 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=685, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,842 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,842 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660836) roll requested 2024-11-19T17:51:00,842 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660842 2024-11-19T17:51:00,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660836 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660842 2024-11-19T17:51:00,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,847 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,848 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,848 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=686, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,848 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660842) roll requested 2024-11-19T17:51:00,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660848 2024-11-19T17:51:00,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,852 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660842 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660848 2024-11-19T17:51:00,853 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,853 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,853 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,853 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=687, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,854 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,854 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660848) roll requested 2024-11-19T17:51:00,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660854 2024-11-19T17:51:00,858 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,858 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,858 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,858 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,858 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660848 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660854 2024-11-19T17:51:00,859 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,859 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,859 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,859 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=688, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,859 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,859 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660854) roll requested 2024-11-19T17:51:00,859 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660859 2024-11-19T17:51:00,864 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,864 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,864 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660854 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660859 2024-11-19T17:51:00,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,865 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,865 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,865 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=689, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,865 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660859) roll requested 2024-11-19T17:51:00,866 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660865 2024-11-19T17:51:00,870 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,870 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,870 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,871 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,871 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660859 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660865 2024-11-19T17:51:00,871 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,871 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,871 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,872 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=690, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,872 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,872 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660865) roll requested 2024-11-19T17:51:00,872 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660872 2024-11-19T17:51:00,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660865 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660872 2024-11-19T17:51:00,877 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,878 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,878 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=691, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,878 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660872) roll requested 2024-11-19T17:51:00,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660878 2024-11-19T17:51:00,882 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,882 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,882 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,882 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,882 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,882 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660872 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660878 2024-11-19T17:51:00,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,883 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,883 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,883 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=692, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,883 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,883 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660878) roll requested 2024-11-19T17:51:00,883 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660883 2024-11-19T17:51:00,888 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,888 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,888 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,888 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,888 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,888 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660878 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660883 2024-11-19T17:51:00,889 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,889 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,889 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,889 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=693, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,889 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,889 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660883) roll requested 2024-11-19T17:51:00,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660889 2024-11-19T17:51:00,894 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,894 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,894 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,894 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,894 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,894 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660883 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660889 2024-11-19T17:51:00,895 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,895 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,895 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,895 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=694, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,895 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,895 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660889) roll requested 2024-11-19T17:51:00,895 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660895 2024-11-19T17:51:00,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,900 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660889 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660895 2024-11-19T17:51:00,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,901 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,901 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=695, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,901 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660895) roll requested 2024-11-19T17:51:00,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660901 2024-11-19T17:51:00,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,906 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,906 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,906 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660895 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660901 2024-11-19T17:51:00,907 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,907 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,907 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,907 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=696, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,907 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,907 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660901) roll requested 2024-11-19T17:51:00,907 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660907 2024-11-19T17:51:00,912 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,912 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,912 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,912 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,912 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,912 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660901 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660907 2024-11-19T17:51:00,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,912 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,912 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,913 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=697, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,913 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,913 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660907) roll requested 2024-11-19T17:51:00,913 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660913 2024-11-19T17:51:00,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,918 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660907 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660913 2024-11-19T17:51:00,918 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,918 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,919 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,919 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=698, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,919 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,919 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660913) roll requested 2024-11-19T17:51:00,919 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660919 2024-11-19T17:51:00,923 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,923 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,923 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,924 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660913 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660919 2024-11-19T17:51:00,924 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,924 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,924 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,925 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=699, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,925 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660919) roll requested 2024-11-19T17:51:00,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660925 2024-11-19T17:51:00,929 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,929 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,929 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,929 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,929 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,930 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660919 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660925 2024-11-19T17:51:00,930 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,930 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,930 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,930 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=700, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,930 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660925) roll requested 2024-11-19T17:51:00,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660931 2024-11-19T17:51:00,935 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,935 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,935 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,936 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,936 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,936 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660925 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660931 2024-11-19T17:51:00,936 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,936 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,936 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,937 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=701, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,937 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660931) roll requested 2024-11-19T17:51:00,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660937 2024-11-19T17:51:00,949 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,949 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,949 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,949 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,950 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,950 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660931 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660937 2024-11-19T17:51:00,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,952 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,953 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=702, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,953 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660937) roll requested 2024-11-19T17:51:00,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660953 2024-11-19T17:51:00,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,958 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,958 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,958 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,958 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:00,958 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660937 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660953 2024-11-19T17:51:00,959 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:00,959 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,959 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:00,959 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660937 is not closed yet, will try archiving it next time 2024-11-19T17:51:00,959 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=703, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:00,959 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:00,959 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660953) roll requested 2024-11-19T17:51:00,959 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038660959 2024-11-19T17:51:01,012 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,012 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,012 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,012 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,012 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,012 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660953 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660959 2024-11-19T17:51:01,013 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,013 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,013 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,014 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=704, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,014 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038660959) roll requested 2024-11-19T17:51:01,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661014 2024-11-19T17:51:01,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038660959 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661014 2024-11-19T17:51:01,019 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,019 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,020 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,020 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=705, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,020 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,020 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661014) roll requested 2024-11-19T17:51:01,020 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661020 2024-11-19T17:51:01,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,025 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661014 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661020 2024-11-19T17:51:01,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,025 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,026 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,026 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=706, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,026 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,026 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661020) roll requested 2024-11-19T17:51:01,026 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661026 2024-11-19T17:51:01,034 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,034 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,034 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,035 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,035 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661020 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661026 2024-11-19T17:51:01,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,036 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,036 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,036 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=707, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,037 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661026) roll requested 2024-11-19T17:51:01,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661037 2024-11-19T17:51:01,041 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,041 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,041 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,041 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,041 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661026 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661037 2024-11-19T17:51:01,042 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,042 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,042 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,042 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=708, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,042 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,042 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661037) roll requested 2024-11-19T17:51:01,042 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661042 2024-11-19T17:51:01,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661037 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661042 2024-11-19T17:51:01,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,048 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,048 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=709, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,048 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661042) roll requested 2024-11-19T17:51:01,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661049 2024-11-19T17:51:01,054 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,054 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,054 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,054 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,054 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,054 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661042 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661049 2024-11-19T17:51:01,055 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,055 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,055 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,055 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=710, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,055 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,055 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661049) roll requested 2024-11-19T17:51:01,055 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661055 2024-11-19T17:51:01,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,060 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,060 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661049 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661055 2024-11-19T17:51:01,062 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,062 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,062 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,062 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=711, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,062 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,062 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661055) roll requested 2024-11-19T17:51:01,062 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661062 2024-11-19T17:51:01,068 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661055 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661062 2024-11-19T17:51:01,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,070 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,070 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=712, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,070 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661062) roll requested 2024-11-19T17:51:01,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661070 2024-11-19T17:51:01,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,077 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,077 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,078 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,078 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661062 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661070 2024-11-19T17:51:01,078 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,078 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,079 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,079 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=713, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,079 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,079 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661070) roll requested 2024-11-19T17:51:01,079 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661079 2024-11-19T17:51:01,086 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,087 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,087 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,087 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,087 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,087 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661070 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661079 2024-11-19T17:51:01,088 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,088 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,088 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,089 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=714, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,089 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661079) roll requested 2024-11-19T17:51:01,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661089 2024-11-19T17:51:01,096 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,096 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,096 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,096 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,096 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,096 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661079 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661089 2024-11-19T17:51:01,097 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,097 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,097 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,098 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=715, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,098 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,098 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661089) roll requested 2024-11-19T17:51:01,098 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661098 2024-11-19T17:51:01,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,104 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661089 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661098 2024-11-19T17:51:01,104 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,105 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,105 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=716, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,105 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661098) roll requested 2024-11-19T17:51:01,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661105 2024-11-19T17:51:01,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,113 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,113 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,113 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,113 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,113 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661098 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661105 2024-11-19T17:51:01,114 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,114 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,114 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,114 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=717, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,114 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,114 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661105) roll requested 2024-11-19T17:51:01,114 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661114 2024-11-19T17:51:01,121 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,121 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,122 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,122 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,122 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,122 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661105 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661114 2024-11-19T17:51:01,123 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,123 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,123 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,123 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=718, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,123 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,123 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661114) roll requested 2024-11-19T17:51:01,123 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661123 2024-11-19T17:51:01,133 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,134 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,134 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,134 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,134 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,134 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661114 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661123 2024-11-19T17:51:01,135 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,135 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,135 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,136 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=719, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,136 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,136 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661123) roll requested 2024-11-19T17:51:01,136 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661136 2024-11-19T17:51:01,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,143 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,143 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,143 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,143 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,143 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661123 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661136 2024-11-19T17:51:01,143 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,143 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,143 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,144 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=720, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,144 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,144 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661136) roll requested 2024-11-19T17:51:01,144 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661144 2024-11-19T17:51:01,148 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,148 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,148 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,148 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,148 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,149 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661136 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661144 2024-11-19T17:51:01,149 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,149 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,149 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=721, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,150 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,150 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661144) roll requested 2024-11-19T17:51:01,150 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661150 2024-11-19T17:51:01,154 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,154 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661144 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661150 2024-11-19T17:51:01,155 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,155 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,156 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=722, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,156 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,156 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661150) roll requested 2024-11-19T17:51:01,156 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661156 2024-11-19T17:51:01,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,160 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,160 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661150 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661156 2024-11-19T17:51:01,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,161 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,161 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=723, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,161 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661156) roll requested 2024-11-19T17:51:01,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661161 2024-11-19T17:51:01,165 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,165 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,165 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,165 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,165 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,165 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661156 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661161 2024-11-19T17:51:01,166 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,166 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,166 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,166 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=724, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,166 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,166 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661161) roll requested 2024-11-19T17:51:01,166 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661166 2024-11-19T17:51:01,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661161 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661166 2024-11-19T17:51:01,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,172 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,172 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=725, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,173 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661166) roll requested 2024-11-19T17:51:01,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661173 2024-11-19T17:51:01,178 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,178 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,178 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,178 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,178 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,178 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661166 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661173 2024-11-19T17:51:01,179 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,179 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,179 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=726, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,179 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661173) roll requested 2024-11-19T17:51:01,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661179 2024-11-19T17:51:01,186 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,186 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,186 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,186 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,186 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,186 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661173 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661179 2024-11-19T17:51:01,187 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,187 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,187 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,187 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=727, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,188 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661179) roll requested 2024-11-19T17:51:01,188 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661188 2024-11-19T17:51:01,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661179 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661188 2024-11-19T17:51:01,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,197 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,197 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=728, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,197 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661188) roll requested 2024-11-19T17:51:01,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661197 2024-11-19T17:51:01,203 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,203 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,203 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,203 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,203 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661188 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661197 2024-11-19T17:51:01,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,204 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,204 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=729, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,204 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,204 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661197) roll requested 2024-11-19T17:51:01,204 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661204 2024-11-19T17:51:01,209 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661197 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661204 2024-11-19T17:51:01,211 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,211 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,212 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=730, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,212 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,212 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661204) roll requested 2024-11-19T17:51:01,212 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661212 2024-11-19T17:51:01,219 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,219 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,219 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,219 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661204 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661212 2024-11-19T17:51:01,220 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,220 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,220 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,220 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=731, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,221 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,221 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661212) roll requested 2024-11-19T17:51:01,221 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661221 2024-11-19T17:51:01,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661212 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661221 2024-11-19T17:51:01,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,229 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,229 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=732, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,230 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,230 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661221) roll requested 2024-11-19T17:51:01,230 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661230 2024-11-19T17:51:01,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661221 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661230 2024-11-19T17:51:01,238 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,238 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,238 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,238 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=733, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,238 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661230) roll requested 2024-11-19T17:51:01,239 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661238 2024-11-19T17:51:01,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661230 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661238 2024-11-19T17:51:01,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,246 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,247 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=734, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,247 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,247 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661238) roll requested 2024-11-19T17:51:01,247 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661247 2024-11-19T17:51:01,253 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,253 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,254 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,254 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,254 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,254 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661238 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661247 2024-11-19T17:51:01,255 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,255 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,258 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,258 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=735, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,258 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,258 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661247) roll requested 2024-11-19T17:51:01,258 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661258 2024-11-19T17:51:01,264 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,264 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,264 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,265 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661247 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661258 2024-11-19T17:51:01,265 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,266 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,266 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,266 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=736, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,266 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,266 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661258) roll requested 2024-11-19T17:51:01,266 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661266 2024-11-19T17:51:01,271 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,272 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,272 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,272 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,272 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,272 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661258 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661266 2024-11-19T17:51:01,273 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,273 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,273 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,273 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=737, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,273 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661266) roll requested 2024-11-19T17:51:01,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661273 2024-11-19T17:51:01,281 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,281 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,281 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,281 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661266 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661273 2024-11-19T17:51:01,282 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,283 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,283 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,283 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=738, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,283 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,283 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661273) roll requested 2024-11-19T17:51:01,283 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661283 2024-11-19T17:51:01,289 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,289 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,289 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,290 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,290 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,290 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661273 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661283 2024-11-19T17:51:01,290 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,290 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,290 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,290 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=739, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,291 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,291 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661283) roll requested 2024-11-19T17:51:01,291 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661291 2024-11-19T17:51:01,295 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,295 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,296 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,296 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,296 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,296 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661283 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661291 2024-11-19T17:51:01,296 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,296 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,296 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,296 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=740, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,297 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,297 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661291) roll requested 2024-11-19T17:51:01,297 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661297 2024-11-19T17:51:01,302 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,302 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,302 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,302 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,302 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,303 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661291 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661297 2024-11-19T17:51:01,303 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,303 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,304 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=741, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,304 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661297) roll requested 2024-11-19T17:51:01,304 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661304 2024-11-19T17:51:01,310 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,310 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,310 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,310 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,310 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661297 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661304 2024-11-19T17:51:01,311 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,311 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,311 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,312 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=742, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,312 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661304) roll requested 2024-11-19T17:51:01,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661312 2024-11-19T17:51:01,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,319 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661304 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661312 2024-11-19T17:51:01,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,320 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,321 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=743, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,321 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,321 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661312) roll requested 2024-11-19T17:51:01,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661321 2024-11-19T17:51:01,328 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,329 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,329 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,329 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661312 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661321 2024-11-19T17:51:01,330 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,330 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,330 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,330 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=744, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,330 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,330 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661321) roll requested 2024-11-19T17:51:01,330 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661330 2024-11-19T17:51:01,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,339 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,339 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,339 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,339 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,340 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661321 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661330 2024-11-19T17:51:01,341 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,341 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,342 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,342 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=745, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,342 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,342 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661330) roll requested 2024-11-19T17:51:01,342 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661342 2024-11-19T17:51:01,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,348 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,348 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,348 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,348 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661330 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661342 2024-11-19T17:51:01,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,348 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,349 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=746, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,349 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661342) roll requested 2024-11-19T17:51:01,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661349 2024-11-19T17:51:01,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,353 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,353 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,353 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661342 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661349 2024-11-19T17:51:01,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,354 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,354 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,354 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=747, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,354 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,354 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661349) roll requested 2024-11-19T17:51:01,354 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661354 2024-11-19T17:51:01,359 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,359 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,359 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,359 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,360 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,360 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661349 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661354 2024-11-19T17:51:01,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,360 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,360 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,360 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=748, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,361 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661354) roll requested 2024-11-19T17:51:01,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661361 2024-11-19T17:51:01,366 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,366 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,366 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,366 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,366 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,366 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661354 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661361 2024-11-19T17:51:01,367 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,367 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,367 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,367 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=749, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,367 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,367 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661361) roll requested 2024-11-19T17:51:01,367 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661367 2024-11-19T17:51:01,373 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,373 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,373 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,373 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661361 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661367 2024-11-19T17:51:01,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,374 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,374 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,374 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=750, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,374 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,374 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661367) roll requested 2024-11-19T17:51:01,375 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661374 2024-11-19T17:51:01,380 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,380 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,380 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,380 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,381 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,381 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661367 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661374 2024-11-19T17:51:01,381 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,381 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,382 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,382 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=751, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,382 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,382 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661374) roll requested 2024-11-19T17:51:01,382 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661382 2024-11-19T17:51:01,389 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661374 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661382 2024-11-19T17:51:01,390 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,390 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,390 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,390 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=752, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,390 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,390 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661382) roll requested 2024-11-19T17:51:01,390 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661390 2024-11-19T17:51:01,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661382 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661390 2024-11-19T17:51:01,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,400 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,400 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=753, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,400 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,401 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661390) roll requested 2024-11-19T17:51:01,401 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661401 2024-11-19T17:51:01,406 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,406 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661390 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661401 2024-11-19T17:51:01,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,408 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,408 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=754, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,408 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,408 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661401) roll requested 2024-11-19T17:51:01,409 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661408 2024-11-19T17:51:01,413 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,413 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661401 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661408 2024-11-19T17:51:01,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,414 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,414 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,414 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=755, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,414 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,414 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661408) roll requested 2024-11-19T17:51:01,414 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661414 2024-11-19T17:51:01,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661408 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661414 2024-11-19T17:51:01,419 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,419 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,420 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=756, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,420 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661414) roll requested 2024-11-19T17:51:01,420 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661420 2024-11-19T17:51:01,424 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,424 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,424 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,424 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,424 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,424 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661414 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661420 2024-11-19T17:51:01,425 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,425 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,425 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,426 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=757, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,426 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,426 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661420) roll requested 2024-11-19T17:51:01,426 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661426 2024-11-19T17:51:01,434 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,434 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,434 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,434 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,434 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,434 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661420 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661426 2024-11-19T17:51:01,435 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,435 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,435 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,435 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=758, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,435 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,435 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661426) roll requested 2024-11-19T17:51:01,435 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661435 2024-11-19T17:51:01,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,441 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661426 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661435 2024-11-19T17:51:01,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,441 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,441 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=759, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,441 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661435) roll requested 2024-11-19T17:51:01,442 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661442 2024-11-19T17:51:01,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,446 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,446 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,447 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661435 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661442 2024-11-19T17:51:01,448 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,448 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,448 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,448 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=760, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,448 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,448 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661442) roll requested 2024-11-19T17:51:01,448 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661448 2024-11-19T17:51:01,453 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,453 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,453 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,453 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,453 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,453 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661442 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661448 2024-11-19T17:51:01,454 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,454 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,454 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,454 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=761, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,454 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,454 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661448) roll requested 2024-11-19T17:51:01,455 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661454 2024-11-19T17:51:01,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,459 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,459 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,459 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661448 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661454 2024-11-19T17:51:01,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,460 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,460 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=762, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,460 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,460 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661454) roll requested 2024-11-19T17:51:01,460 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661460 2024-11-19T17:51:01,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661454 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661460 2024-11-19T17:51:01,466 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,466 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,466 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=763, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,466 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661460) roll requested 2024-11-19T17:51:01,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661466 2024-11-19T17:51:01,472 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,472 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,472 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661460 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661466 2024-11-19T17:51:01,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,473 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,474 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=764, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,474 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,474 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661466) roll requested 2024-11-19T17:51:01,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661474 2024-11-19T17:51:01,478 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,478 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,478 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,478 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,478 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,478 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661466 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661474 2024-11-19T17:51:01,479 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,479 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,479 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,479 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=765, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,479 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,479 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661474) roll requested 2024-11-19T17:51:01,479 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661479 2024-11-19T17:51:01,484 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,484 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,484 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,484 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,484 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,484 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661474 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661479 2024-11-19T17:51:01,485 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,485 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,485 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,485 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=766, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,485 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661479) roll requested 2024-11-19T17:51:01,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661485 2024-11-19T17:51:01,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661479 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661485 2024-11-19T17:51:01,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,490 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,490 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=767, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,490 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,490 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661485) roll requested 2024-11-19T17:51:01,491 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661491 2024-11-19T17:51:01,497 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,497 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,497 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,497 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,497 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,497 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661485 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661491 2024-11-19T17:51:01,498 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,498 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,498 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,498 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=768, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,498 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,498 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661491) roll requested 2024-11-19T17:51:01,498 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661498 2024-11-19T17:51:01,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661491 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661498 2024-11-19T17:51:01,503 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,503 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,504 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=769, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,504 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661498) roll requested 2024-11-19T17:51:01,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661504 2024-11-19T17:51:01,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,508 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,508 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,508 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,508 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,508 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661498 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661504 2024-11-19T17:51:01,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,508 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,508 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=770, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,509 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,509 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661504) roll requested 2024-11-19T17:51:01,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661509 2024-11-19T17:51:01,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661504 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661509 2024-11-19T17:51:01,515 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,515 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,515 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=771, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,515 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661509) roll requested 2024-11-19T17:51:01,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661515 2024-11-19T17:51:01,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,522 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661509 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661515 2024-11-19T17:51:01,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,522 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,523 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=772, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,523 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,523 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661515) roll requested 2024-11-19T17:51:01,523 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661523 2024-11-19T17:51:01,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,530 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,530 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,530 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,530 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661515 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661523 2024-11-19T17:51:01,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,531 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,531 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=773, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,531 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661523) roll requested 2024-11-19T17:51:01,532 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661532 2024-11-19T17:51:01,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,539 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661523 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661532 2024-11-19T17:51:01,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,540 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,540 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,540 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=774, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,540 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,540 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661532) roll requested 2024-11-19T17:51:01,540 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661540 2024-11-19T17:51:01,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,548 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,548 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661532 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661540 2024-11-19T17:51:01,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,549 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,549 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=775, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,549 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661540) roll requested 2024-11-19T17:51:01,550 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661550 2024-11-19T17:51:01,556 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,556 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,556 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,556 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,556 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,557 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661540 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661550 2024-11-19T17:51:01,558 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,558 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,558 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,558 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=776, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,558 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,558 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661550) roll requested 2024-11-19T17:51:01,558 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661558 2024-11-19T17:51:01,566 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,566 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,566 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,566 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,566 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,566 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661550 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661558 2024-11-19T17:51:01,567 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,567 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,567 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,567 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=777, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,568 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,568 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661558) roll requested 2024-11-19T17:51:01,568 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661568 2024-11-19T17:51:01,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,574 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,574 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661558 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661568 2024-11-19T17:51:01,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,576 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,576 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=778, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,576 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661568) roll requested 2024-11-19T17:51:01,576 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661576 2024-11-19T17:51:01,582 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,582 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,582 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,582 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,582 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,582 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661568 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661576 2024-11-19T17:51:01,583 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,583 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,583 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,584 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=779, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,584 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661576) roll requested 2024-11-19T17:51:01,584 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661584 2024-11-19T17:51:01,589 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,589 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,589 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,589 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,589 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,589 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661576 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661584 2024-11-19T17:51:01,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,592 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,592 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=780, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,592 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661584) roll requested 2024-11-19T17:51:01,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661593 2024-11-19T17:51:01,597 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,597 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,597 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,597 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,597 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,597 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661584 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661593 2024-11-19T17:51:01,597 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,597 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,598 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,598 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=781, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,598 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,598 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661593) roll requested 2024-11-19T17:51:01,598 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661598 2024-11-19T17:51:01,608 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,608 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,608 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,608 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661593 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661598 2024-11-19T17:51:01,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,609 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,609 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,609 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=782, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,609 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661598) roll requested 2024-11-19T17:51:01,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661609 2024-11-19T17:51:01,613 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,613 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,613 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,613 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,613 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661598 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661609 2024-11-19T17:51:01,616 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,616 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,616 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,616 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=783, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,616 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,616 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661609) roll requested 2024-11-19T17:51:01,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661616 2024-11-19T17:51:01,621 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,621 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,621 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,621 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,621 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,621 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661609 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661616 2024-11-19T17:51:01,622 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,622 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,622 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,622 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=784, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,622 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,622 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661616) roll requested 2024-11-19T17:51:01,622 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661622 2024-11-19T17:51:01,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,626 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,626 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,626 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,626 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,626 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661616 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661622 2024-11-19T17:51:01,627 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,627 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,627 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,627 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=785, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,627 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,627 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661622) roll requested 2024-11-19T17:51:01,628 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661627 2024-11-19T17:51:01,632 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,632 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,632 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,632 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,632 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,632 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661622 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661627 2024-11-19T17:51:01,633 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,633 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,633 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,633 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=786, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,633 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,633 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661627) roll requested 2024-11-19T17:51:01,633 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661633 2024-11-19T17:51:01,638 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,638 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,638 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661627 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661633 2024-11-19T17:51:01,639 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,639 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,639 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,639 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=787, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,639 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,639 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661633) roll requested 2024-11-19T17:51:01,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661639 2024-11-19T17:51:01,645 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,646 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,646 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,646 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,646 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,646 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661633 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661639 2024-11-19T17:51:01,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,647 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,647 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,647 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=788, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,647 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,647 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661639) roll requested 2024-11-19T17:51:01,647 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661647 2024-11-19T17:51:01,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,652 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661639 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661647 2024-11-19T17:51:01,653 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,653 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,653 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,654 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=789, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,654 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661647) roll requested 2024-11-19T17:51:01,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661654 2024-11-19T17:51:01,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,662 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661647 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661654 2024-11-19T17:51:01,663 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,663 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,663 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,663 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=790, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,663 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,663 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661654) roll requested 2024-11-19T17:51:01,663 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661663 2024-11-19T17:51:01,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,671 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,671 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,671 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,671 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661654 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661663 2024-11-19T17:51:01,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,672 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,673 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,673 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=791, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,673 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,673 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661663) roll requested 2024-11-19T17:51:01,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661673 2024-11-19T17:51:01,679 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,679 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,679 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,679 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,679 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661663 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661673 2024-11-19T17:51:01,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,680 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,681 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=792, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,681 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661673) roll requested 2024-11-19T17:51:01,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661681 2024-11-19T17:51:01,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661673 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661681 2024-11-19T17:51:01,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,689 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,690 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=793, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,690 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,690 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661681) roll requested 2024-11-19T17:51:01,690 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661690 2024-11-19T17:51:01,698 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,698 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,698 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,698 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,698 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,698 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661681 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661690 2024-11-19T17:51:01,698 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,699 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,699 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,699 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=794, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,699 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,699 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661690) roll requested 2024-11-19T17:51:01,699 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661699 2024-11-19T17:51:01,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661690 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661699 2024-11-19T17:51:01,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,704 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,704 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=795, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,704 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661699) roll requested 2024-11-19T17:51:01,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661704 2024-11-19T17:51:01,709 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,709 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,709 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,709 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,709 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,709 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661699 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661704 2024-11-19T17:51:01,710 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,710 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,710 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,710 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=796, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,710 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,710 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661704) roll requested 2024-11-19T17:51:01,710 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661710 2024-11-19T17:51:01,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661704 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661710 2024-11-19T17:51:01,715 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,715 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,715 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,716 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=797, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,716 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,716 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661710) roll requested 2024-11-19T17:51:01,716 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661716 2024-11-19T17:51:01,720 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,720 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,720 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,720 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,720 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,720 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661710 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661716 2024-11-19T17:51:01,721 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,721 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,721 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,721 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=798, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,721 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661716) roll requested 2024-11-19T17:51:01,722 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661722 2024-11-19T17:51:01,727 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,727 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661716 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661722 2024-11-19T17:51:01,728 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,728 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,728 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,728 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=799, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,728 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,728 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661722) roll requested 2024-11-19T17:51:01,729 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661729 2024-11-19T17:51:01,734 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,735 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,735 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661722 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661729 2024-11-19T17:51:01,735 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,736 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,736 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=800, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,737 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661729) roll requested 2024-11-19T17:51:01,737 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661737 2024-11-19T17:51:01,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,744 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,744 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661729 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661737 2024-11-19T17:51:01,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,748 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,749 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=801, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,749 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661737) roll requested 2024-11-19T17:51:01,749 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661749 2024-11-19T17:51:01,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661737 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661749 2024-11-19T17:51:01,755 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,756 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,756 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=802, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,756 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661749) roll requested 2024-11-19T17:51:01,756 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661756 2024-11-19T17:51:01,760 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,760 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,761 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,761 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,761 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,761 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661749 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661756 2024-11-19T17:51:01,761 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,761 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,762 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,762 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=803, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,762 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,762 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661756) roll requested 2024-11-19T17:51:01,762 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661762 2024-11-19T17:51:01,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661756 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661762 2024-11-19T17:51:01,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,768 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=804, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,768 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,768 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661762) roll requested 2024-11-19T17:51:01,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661768 2024-11-19T17:51:01,774 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,774 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,774 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,774 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,774 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,774 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661762 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661768 2024-11-19T17:51:01,775 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,775 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,775 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,775 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=805, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,775 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,775 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661768) roll requested 2024-11-19T17:51:01,776 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661775 2024-11-19T17:51:01,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,782 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,782 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,782 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,782 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,782 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661768 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661775 2024-11-19T17:51:01,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,784 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,784 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=806, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,784 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661775) roll requested 2024-11-19T17:51:01,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661784 2024-11-19T17:51:01,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,792 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,792 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,792 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,792 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661775 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661784 2024-11-19T17:51:01,793 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,793 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,793 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=807, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,793 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,793 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661784) roll requested 2024-11-19T17:51:01,794 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661794 2024-11-19T17:51:01,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661784 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661794 2024-11-19T17:51:01,802 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,802 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,802 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,802 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=808, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,803 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,803 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661794) roll requested 2024-11-19T17:51:01,803 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661803 2024-11-19T17:51:01,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661794 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661803 2024-11-19T17:51:01,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,812 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,813 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=809, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,813 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661803) roll requested 2024-11-19T17:51:01,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661813 2024-11-19T17:51:01,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,821 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661803 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661813 2024-11-19T17:51:01,822 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,822 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,823 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=810, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,823 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,823 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661813) roll requested 2024-11-19T17:51:01,823 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661823 2024-11-19T17:51:01,830 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,830 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661813 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661823 2024-11-19T17:51:01,832 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,832 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,832 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,832 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=811, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,832 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,832 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661823) roll requested 2024-11-19T17:51:01,832 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661832 2024-11-19T17:51:01,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,837 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,837 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,837 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,837 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661823 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661832 2024-11-19T17:51:01,837 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,837 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,837 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,838 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=812, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,838 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,838 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661832) roll requested 2024-11-19T17:51:01,838 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661838 2024-11-19T17:51:01,842 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,842 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,842 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,842 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,843 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,843 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661832 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661838 2024-11-19T17:51:01,844 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,844 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,844 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,844 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=813, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,844 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,844 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661838) roll requested 2024-11-19T17:51:01,844 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661844 2024-11-19T17:51:01,850 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,850 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,850 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,850 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,850 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,850 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661838 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661844 2024-11-19T17:51:01,851 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,851 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,851 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,851 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=814, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,852 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,852 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661844) roll requested 2024-11-19T17:51:01,852 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661852 2024-11-19T17:51:01,858 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,858 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,858 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,858 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,858 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,858 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661844 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661852 2024-11-19T17:51:01,859 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,859 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,859 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,860 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=815, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,860 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661852) roll requested 2024-11-19T17:51:01,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661860 2024-11-19T17:51:01,867 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,867 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,867 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,867 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661852 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661860 2024-11-19T17:51:01,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,868 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,869 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=816, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,869 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,869 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661860) roll requested 2024-11-19T17:51:01,869 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661869 2024-11-19T17:51:01,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,874 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,874 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,874 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,874 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,874 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661860 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661869 2024-11-19T17:51:01,875 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,875 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,875 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,875 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=817, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,875 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,875 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661869) roll requested 2024-11-19T17:51:01,875 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661875 2024-11-19T17:51:01,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,881 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,881 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,881 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,881 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,881 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661869 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661875 2024-11-19T17:51:01,882 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,882 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,882 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,882 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=818, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,882 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,882 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661875) roll requested 2024-11-19T17:51:01,882 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661882 2024-11-19T17:51:01,887 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,888 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,888 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,888 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,888 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,888 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661875 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661882 2024-11-19T17:51:01,889 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,889 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,889 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,889 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=819, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,889 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,889 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661882) roll requested 2024-11-19T17:51:01,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661889 2024-11-19T17:51:01,896 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,896 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,896 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,896 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,896 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,896 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661882 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661889 2024-11-19T17:51:01,897 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,897 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,897 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,897 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=820, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,897 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,897 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661889) roll requested 2024-11-19T17:51:01,898 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661897 2024-11-19T17:51:01,904 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,904 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,904 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,905 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661889 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661897 2024-11-19T17:51:01,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,906 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,906 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,906 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=821, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,906 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,906 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661897) roll requested 2024-11-19T17:51:01,907 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661907 2024-11-19T17:51:01,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,913 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,914 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,914 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661897 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661907 2024-11-19T17:51:01,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,915 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,915 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,915 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=822, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,915 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,915 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661907) roll requested 2024-11-19T17:51:01,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661915 2024-11-19T17:51:01,922 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,922 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,922 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,922 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,922 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,923 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661907 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661915 2024-11-19T17:51:01,923 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,923 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,923 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,924 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=823, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,924 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661915) roll requested 2024-11-19T17:51:01,924 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661924 2024-11-19T17:51:01,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,930 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,930 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,931 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661915 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661924 2024-11-19T17:51:01,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,932 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,933 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=824, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,933 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661924) roll requested 2024-11-19T17:51:01,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661933 2024-11-19T17:51:01,940 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,940 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,941 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,941 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,941 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,941 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661924 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661933 2024-11-19T17:51:01,942 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,942 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,943 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,943 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=825, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,943 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,944 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661933) roll requested 2024-11-19T17:51:01,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661944 2024-11-19T17:51:01,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661933 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661944 2024-11-19T17:51:01,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,952 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,953 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=826, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,953 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661944) roll requested 2024-11-19T17:51:01,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661953 2024-11-19T17:51:01,960 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,961 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,961 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,961 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,961 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,961 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661944 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661953 2024-11-19T17:51:01,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,964 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,964 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=827, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,964 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661953) roll requested 2024-11-19T17:51:01,964 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661964 2024-11-19T17:51:01,970 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,970 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661953 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661964 2024-11-19T17:51:01,971 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,971 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,971 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,971 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=828, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,971 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,971 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661964) roll requested 2024-11-19T17:51:01,971 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661971 2024-11-19T17:51:01,975 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,975 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,976 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661964 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661971 2024-11-19T17:51:01,976 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,976 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,976 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,977 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=829, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,977 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661971) roll requested 2024-11-19T17:51:01,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661977 2024-11-19T17:51:01,984 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,984 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,984 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,984 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,984 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,985 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661971 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661977 2024-11-19T17:51:01,985 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,985 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,986 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=830, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,986 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,986 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661977) roll requested 2024-11-19T17:51:01,986 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661986 2024-11-19T17:51:01,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661977 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661986 2024-11-19T17:51:01,991 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,991 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,991 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=831, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,991 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,991 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661986) roll requested 2024-11-19T17:51:01,992 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661992 2024-11-19T17:51:01,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,996 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,996 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,997 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:01,997 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661986 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661992 2024-11-19T17:51:01,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:01,997 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,998 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:01,998 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=832, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:01,998 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:01,998 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661992) roll requested 2024-11-19T17:51:01,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038661998 2024-11-19T17:51:02,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,002 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,002 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,002 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,002 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,002 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661992 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661998 2024-11-19T17:51:02,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,003 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,003 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,004 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=833, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,004 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038661998) roll requested 2024-11-19T17:51:02,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662004 2024-11-19T17:51:02,009 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,009 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,009 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038661998 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662004 2024-11-19T17:51:02,010 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,010 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,010 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,010 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=834, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,010 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,010 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662004) roll requested 2024-11-19T17:51:02,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662011 2024-11-19T17:51:02,017 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,017 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662004 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662011 2024-11-19T17:51:02,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,019 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,019 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,019 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=835, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,019 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,019 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662011) roll requested 2024-11-19T17:51:02,019 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662019 2024-11-19T17:51:02,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,027 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,027 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,027 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,027 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,027 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662011 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662019 2024-11-19T17:51:02,029 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,029 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,029 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,029 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=836, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,029 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,029 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662019) roll requested 2024-11-19T17:51:02,029 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662029 2024-11-19T17:51:02,033 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,033 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,034 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,034 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,034 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,034 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662019 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662029 2024-11-19T17:51:02,035 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,035 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,035 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,035 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=837, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,035 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,035 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662029) roll requested 2024-11-19T17:51:02,035 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662035 2024-11-19T17:51:02,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,040 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662029 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662035 2024-11-19T17:51:02,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,041 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,041 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=838, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,041 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662035) roll requested 2024-11-19T17:51:02,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662041 2024-11-19T17:51:02,046 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,046 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662035 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662041 2024-11-19T17:51:02,049 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,049 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,049 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,050 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=839, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,050 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,050 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662041) roll requested 2024-11-19T17:51:02,050 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662050 2024-11-19T17:51:02,054 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,054 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,054 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,055 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,055 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662041 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662050 2024-11-19T17:51:02,055 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,055 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,055 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,055 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=840, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,056 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662050) roll requested 2024-11-19T17:51:02,056 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662056 2024-11-19T17:51:02,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,060 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,060 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,060 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,060 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,060 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662050 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662056 2024-11-19T17:51:02,060 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,060 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,060 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,061 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=841, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,061 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,061 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662056) roll requested 2024-11-19T17:51:02,061 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662061 2024-11-19T17:51:02,066 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,066 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,066 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,066 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,066 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,066 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662056 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662061 2024-11-19T17:51:02,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,068 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,068 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,069 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=842, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,069 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662061) roll requested 2024-11-19T17:51:02,069 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662069 2024-11-19T17:51:02,074 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,074 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,074 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,074 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,074 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,074 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662061 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662069 2024-11-19T17:51:02,075 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,075 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,075 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,075 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=843, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,076 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,076 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662069) roll requested 2024-11-19T17:51:02,076 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662076 2024-11-19T17:51:02,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,083 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,083 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,083 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662069 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662076 2024-11-19T17:51:02,084 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,084 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,084 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,084 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=844, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,084 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,084 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662076) roll requested 2024-11-19T17:51:02,084 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662084 2024-11-19T17:51:02,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,091 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,091 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662076 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662084 2024-11-19T17:51:02,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,093 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=845, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,092 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,093 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,093 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662084) roll requested 2024-11-19T17:51:02,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662093 2024-11-19T17:51:02,099 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,099 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,099 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,099 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,099 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,099 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662084 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662093 2024-11-19T17:51:02,100 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,100 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,100 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,100 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=846, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,100 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,101 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662093) roll requested 2024-11-19T17:51:02,101 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662101 2024-11-19T17:51:02,107 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,107 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,107 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,107 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,107 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,107 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662093 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662101 2024-11-19T17:51:02,108 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,108 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,108 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,108 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=847, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,108 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,108 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662101) roll requested 2024-11-19T17:51:02,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662109 2024-11-19T17:51:02,115 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,115 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,116 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,116 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,116 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662101 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662109 2024-11-19T17:51:02,117 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,117 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,117 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,117 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=848, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,117 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,117 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662109) roll requested 2024-11-19T17:51:02,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662117 2024-11-19T17:51:02,123 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,123 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662109 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662117 2024-11-19T17:51:02,124 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,124 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,124 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,124 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=849, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,124 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,124 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662117) roll requested 2024-11-19T17:51:02,124 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662124 2024-11-19T17:51:02,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662117 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662124 2024-11-19T17:51:02,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,129 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,129 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=850, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,129 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662124) roll requested 2024-11-19T17:51:02,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662129 2024-11-19T17:51:02,134 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,134 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,134 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,134 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,134 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,134 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662124 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662129 2024-11-19T17:51:02,135 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,135 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,135 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,135 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=851, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,136 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,136 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662129) roll requested 2024-11-19T17:51:02,136 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662136 2024-11-19T17:51:02,141 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,141 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,141 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,141 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,141 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,141 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662129 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662136 2024-11-19T17:51:02,142 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,142 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,142 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,142 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=852, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,142 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,142 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662136) roll requested 2024-11-19T17:51:02,142 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662142 2024-11-19T17:51:02,148 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,148 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,148 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,148 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,149 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,149 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662136 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662142 2024-11-19T17:51:02,149 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,149 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,150 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=853, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,150 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,150 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662142) roll requested 2024-11-19T17:51:02,150 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662150 2024-11-19T17:51:02,155 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,156 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,156 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,156 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,156 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,156 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662142 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662150 2024-11-19T17:51:02,157 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,157 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,157 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,157 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=854, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,157 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662150) roll requested 2024-11-19T17:51:02,158 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662158 2024-11-19T17:51:02,165 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,165 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,166 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,166 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,166 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,166 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662150 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662158 2024-11-19T17:51:02,167 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,167 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,167 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,167 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=855, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,168 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,168 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662158) roll requested 2024-11-19T17:51:02,168 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662168 2024-11-19T17:51:02,174 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,174 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,174 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,174 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,174 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,174 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662158 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662168 2024-11-19T17:51:02,175 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,175 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,175 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,175 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=856, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,175 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,175 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662168) roll requested 2024-11-19T17:51:02,175 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662175 2024-11-19T17:51:02,181 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,182 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,182 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662168 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662175 2024-11-19T17:51:02,183 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,183 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,183 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,183 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=857, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,183 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,183 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662175) roll requested 2024-11-19T17:51:02,184 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662184 2024-11-19T17:51:02,190 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,191 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,191 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,191 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,191 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,191 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662175 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662184 2024-11-19T17:51:02,194 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,194 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,194 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,195 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=858, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,195 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,195 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662184) roll requested 2024-11-19T17:51:02,195 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662195 2024-11-19T17:51:02,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,202 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,202 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662184 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662195 2024-11-19T17:51:02,203 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,203 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,203 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,203 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=859, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,203 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,203 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662195) roll requested 2024-11-19T17:51:02,203 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662203 2024-11-19T17:51:02,209 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662195 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662203 2024-11-19T17:51:02,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,211 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,211 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=860, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,211 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,211 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662203) roll requested 2024-11-19T17:51:02,211 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662211 2024-11-19T17:51:02,216 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,216 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,217 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,217 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,217 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,217 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662203 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662211 2024-11-19T17:51:02,218 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,218 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,218 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,218 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=861, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,218 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,218 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662211) roll requested 2024-11-19T17:51:02,218 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662218 2024-11-19T17:51:02,224 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,224 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,225 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,225 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662211 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662218 2024-11-19T17:51:02,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,226 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,226 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,226 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=862, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,226 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662218) roll requested 2024-11-19T17:51:02,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662226 2024-11-19T17:51:02,233 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,234 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,234 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,234 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,234 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,234 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662218 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662226 2024-11-19T17:51:02,235 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,235 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,235 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,235 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=863, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,235 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,235 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662226) roll requested 2024-11-19T17:51:02,236 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662235 2024-11-19T17:51:02,242 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,242 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,242 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,242 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,242 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,242 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662226 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662235 2024-11-19T17:51:02,243 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,243 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,243 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,244 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=864, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,244 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,244 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662235) roll requested 2024-11-19T17:51:02,244 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662244 2024-11-19T17:51:02,251 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,251 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,251 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,251 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662235 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662244 2024-11-19T17:51:02,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,252 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,252 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=865, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,253 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662244) roll requested 2024-11-19T17:51:02,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662253 2024-11-19T17:51:02,257 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,257 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662244 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662253 2024-11-19T17:51:02,258 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,258 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,258 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,259 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=866, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,259 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662253) roll requested 2024-11-19T17:51:02,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662259 2024-11-19T17:51:02,265 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,265 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,265 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,265 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662253 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662259 2024-11-19T17:51:02,266 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,266 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,266 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,267 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=867, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,267 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,267 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662259) roll requested 2024-11-19T17:51:02,267 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662267 2024-11-19T17:51:02,273 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,273 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,273 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,273 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,273 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,273 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662259 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662267 2024-11-19T17:51:02,274 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,274 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,274 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,274 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=868, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,274 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,274 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662267) roll requested 2024-11-19T17:51:02,274 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662274 2024-11-19T17:51:02,280 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,281 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,281 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,281 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662267 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662274 2024-11-19T17:51:02,282 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,282 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,282 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,282 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=869, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,282 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,282 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662274) roll requested 2024-11-19T17:51:02,282 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662282 2024-11-19T17:51:02,288 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,288 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,289 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,289 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,289 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,289 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662274 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662282 2024-11-19T17:51:02,290 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,290 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,290 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,290 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=870, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,291 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,291 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662282) roll requested 2024-11-19T17:51:02,291 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662291 2024-11-19T17:51:02,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,299 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662282 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662291 2024-11-19T17:51:02,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,301 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,301 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=871, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,301 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662291) roll requested 2024-11-19T17:51:02,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662301 2024-11-19T17:51:02,309 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,309 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,309 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,309 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,309 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,309 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662291 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662301 2024-11-19T17:51:02,310 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,310 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,310 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,311 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=872, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,311 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,311 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662301) roll requested 2024-11-19T17:51:02,311 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662311 2024-11-19T17:51:02,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,318 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,318 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,318 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662301 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662311 2024-11-19T17:51:02,319 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,319 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,319 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,319 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=873, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,319 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,319 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662311) roll requested 2024-11-19T17:51:02,319 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662319 2024-11-19T17:51:02,326 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,327 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,327 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,327 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,327 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,327 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662311 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662319 2024-11-19T17:51:02,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,328 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,328 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,328 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=874, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,328 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662319) roll requested 2024-11-19T17:51:02,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662329 2024-11-19T17:51:02,337 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,337 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,338 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662319 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662329 2024-11-19T17:51:02,339 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,339 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,339 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=875, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,339 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662329) roll requested 2024-11-19T17:51:02,340 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662339 2024-11-19T17:51:02,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,348 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,348 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,348 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,348 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662329 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662339 2024-11-19T17:51:02,349 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,349 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,349 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=876, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,350 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662339) roll requested 2024-11-19T17:51:02,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662350 2024-11-19T17:51:02,356 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,356 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,356 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,357 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,357 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,357 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662339 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662350 2024-11-19T17:51:02,358 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,358 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,358 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,358 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=877, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,358 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,358 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662350) roll requested 2024-11-19T17:51:02,358 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662358 2024-11-19T17:51:02,364 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,364 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,364 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,364 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,365 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662350 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662358 2024-11-19T17:51:02,365 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,365 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,365 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,366 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=878, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,366 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,366 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662358) roll requested 2024-11-19T17:51:02,366 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662366 2024-11-19T17:51:02,372 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,372 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,372 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,372 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,372 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,372 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662358 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662366 2024-11-19T17:51:02,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,373 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,373 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,373 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=879, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,373 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,373 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662366) roll requested 2024-11-19T17:51:02,373 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662373 2024-11-19T17:51:02,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,378 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662366 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662373 2024-11-19T17:51:02,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,379 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,379 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,379 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=880, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,380 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662373) roll requested 2024-11-19T17:51:02,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662380 2024-11-19T17:51:02,386 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,386 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,386 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,386 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662373 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662380 2024-11-19T17:51:02,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,387 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,387 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=881, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,387 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662380) roll requested 2024-11-19T17:51:02,387 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662387 2024-11-19T17:51:02,394 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,394 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,394 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,394 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,395 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,395 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662380 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662387 2024-11-19T17:51:02,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,396 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,396 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=882, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,396 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,396 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662387) roll requested 2024-11-19T17:51:02,396 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662396 2024-11-19T17:51:02,404 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,404 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,404 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,404 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,404 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662387 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662396 2024-11-19T17:51:02,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,405 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,406 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=883, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,406 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662396) roll requested 2024-11-19T17:51:02,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662406 2024-11-19T17:51:02,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,415 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662396 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662406 2024-11-19T17:51:02,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,415 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,416 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=884, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,416 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,416 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662406) roll requested 2024-11-19T17:51:02,416 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662416 2024-11-19T17:51:02,422 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662406 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662416 2024-11-19T17:51:02,423 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,423 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,423 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,424 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=885, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,424 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662416) roll requested 2024-11-19T17:51:02,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662424 2024-11-19T17:51:02,429 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,429 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,430 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662416 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662424 2024-11-19T17:51:02,430 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,430 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,431 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,431 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=886, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,431 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,431 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662424) roll requested 2024-11-19T17:51:02,431 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662431 2024-11-19T17:51:02,438 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,438 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,438 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,438 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,438 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,438 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662424 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662431 2024-11-19T17:51:02,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,439 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,439 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=887, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,440 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662431) roll requested 2024-11-19T17:51:02,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662440 2024-11-19T17:51:02,447 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,447 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,447 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,448 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662431 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662440 2024-11-19T17:51:02,448 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,448 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,448 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,449 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=888, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,449 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,449 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662440) roll requested 2024-11-19T17:51:02,449 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662449 2024-11-19T17:51:02,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662440 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662449 2024-11-19T17:51:02,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,457 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,458 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=889, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,458 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,458 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662449) roll requested 2024-11-19T17:51:02,458 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662458 2024-11-19T17:51:02,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662449 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662458 2024-11-19T17:51:02,466 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,466 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,467 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=890, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,467 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662458) roll requested 2024-11-19T17:51:02,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662467 2024-11-19T17:51:02,476 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,476 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,476 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,476 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,476 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,476 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662458 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662467 2024-11-19T17:51:02,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,477 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,477 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,478 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=891, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,478 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662467) roll requested 2024-11-19T17:51:02,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662478 2024-11-19T17:51:02,485 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,485 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,485 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,485 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,486 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662467 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662478 2024-11-19T17:51:02,487 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,487 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,487 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,487 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=892, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,487 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662478) roll requested 2024-11-19T17:51:02,487 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662487 2024-11-19T17:51:02,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,496 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,496 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,496 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662478 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662487 2024-11-19T17:51:02,497 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,497 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,497 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=893, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,497 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662487) roll requested 2024-11-19T17:51:02,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662497 2024-11-19T17:51:02,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,502 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,502 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662487 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662497 2024-11-19T17:51:02,503 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,503 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,504 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=894, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,504 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662497) roll requested 2024-11-19T17:51:02,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662504 2024-11-19T17:51:02,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,510 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,510 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,510 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,510 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662497 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662504 2024-11-19T17:51:02,511 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,511 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,511 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,511 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=895, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,511 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,511 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662504) roll requested 2024-11-19T17:51:02,511 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662511 2024-11-19T17:51:02,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,518 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,518 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662504 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662511 2024-11-19T17:51:02,519 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,519 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,519 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,519 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=896, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,519 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,519 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662511) roll requested 2024-11-19T17:51:02,520 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662519 2024-11-19T17:51:02,527 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662511 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662519 2024-11-19T17:51:02,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,528 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,529 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=897, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,529 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,529 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662519) roll requested 2024-11-19T17:51:02,529 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662529 2024-11-19T17:51:02,536 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,536 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,537 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,537 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,537 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662519 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662529 2024-11-19T17:51:02,538 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,538 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,538 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,538 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=898, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,538 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,538 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662529) roll requested 2024-11-19T17:51:02,538 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662538 2024-11-19T17:51:02,545 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,546 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,546 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,546 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662529 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662538 2024-11-19T17:51:02,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,552 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,553 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=899, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,553 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662538) roll requested 2024-11-19T17:51:02,553 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662553 2024-11-19T17:51:02,559 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,560 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,560 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,560 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,560 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,560 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662538 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662553 2024-11-19T17:51:02,560 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,561 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,561 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,561 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=900, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,561 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,561 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662553) roll requested 2024-11-19T17:51:02,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662561 2024-11-19T17:51:02,565 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,565 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,565 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,565 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,566 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,566 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662553 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662561 2024-11-19T17:51:02,566 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,566 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,566 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,567 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=901, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,567 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,567 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662561) roll requested 2024-11-19T17:51:02,567 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662567 2024-11-19T17:51:02,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662561 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662567 2024-11-19T17:51:02,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,572 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,573 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=902, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,573 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,573 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662567) roll requested 2024-11-19T17:51:02,573 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662573 2024-11-19T17:51:02,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,578 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,578 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,578 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,578 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662567 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662573 2024-11-19T17:51:02,579 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,579 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,579 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,579 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=903, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,579 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,579 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662573) roll requested 2024-11-19T17:51:02,579 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662579 2024-11-19T17:51:02,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,586 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,586 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,586 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,586 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,586 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662573 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662579 2024-11-19T17:51:02,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,587 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,588 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=904, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,588 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,588 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662579) roll requested 2024-11-19T17:51:02,588 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662588 2024-11-19T17:51:02,594 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,594 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,595 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,595 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662579 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662588 2024-11-19T17:51:02,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,596 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,596 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,596 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=905, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,596 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,596 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662588) roll requested 2024-11-19T17:51:02,596 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662596 2024-11-19T17:51:02,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662588 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662596 2024-11-19T17:51:02,603 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,603 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,603 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,603 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=906, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,603 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,603 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662596) roll requested 2024-11-19T17:51:02,603 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662603 2024-11-19T17:51:02,610 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,610 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,610 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,610 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,610 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662596 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662603 2024-11-19T17:51:02,611 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,611 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,611 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,612 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=907, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,612 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662603) roll requested 2024-11-19T17:51:02,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662612 2024-11-19T17:51:02,619 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,619 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,619 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,619 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,619 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,620 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662603 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662612 2024-11-19T17:51:02,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,620 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,621 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=908, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,621 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662612) roll requested 2024-11-19T17:51:02,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662621 2024-11-19T17:51:02,628 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,628 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662612 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662621 2024-11-19T17:51:02,629 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,629 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,629 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=909, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,629 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662621) roll requested 2024-11-19T17:51:02,630 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662630 2024-11-19T17:51:02,636 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,636 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,636 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,636 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,636 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,637 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662621 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662630 2024-11-19T17:51:02,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,637 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,638 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,638 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=910, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,638 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,638 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662630) roll requested 2024-11-19T17:51:02,638 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662638 2024-11-19T17:51:02,645 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,645 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,645 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,645 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662630 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662638 2024-11-19T17:51:02,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,646 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,646 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,646 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=911, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,646 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,646 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662638) roll requested 2024-11-19T17:51:02,646 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662646 2024-11-19T17:51:02,651 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,651 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,651 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662638 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662646 2024-11-19T17:51:02,651 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,651 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,652 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,652 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=912, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,652 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,652 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662646) roll requested 2024-11-19T17:51:02,652 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662652 2024-11-19T17:51:02,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,657 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,657 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,657 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,657 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,658 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662646 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662652 2024-11-19T17:51:02,658 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,658 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,658 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,659 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=913, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,659 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,659 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662652) roll requested 2024-11-19T17:51:02,659 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662659 2024-11-19T17:51:02,664 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,664 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,665 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,665 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,665 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,665 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662652 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662659 2024-11-19T17:51:02,665 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,665 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,666 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,666 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=914, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,666 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,666 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662659) roll requested 2024-11-19T17:51:02,666 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662666 2024-11-19T17:51:02,672 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,673 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,673 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662659 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662666 2024-11-19T17:51:02,675 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,675 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,675 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,675 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=915, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,675 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,675 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662666) roll requested 2024-11-19T17:51:02,675 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662675 2024-11-19T17:51:02,681 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,681 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,681 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,681 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,681 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,681 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662666 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662675 2024-11-19T17:51:02,682 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,682 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,682 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,682 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=916, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,682 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,682 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662675) roll requested 2024-11-19T17:51:02,682 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662682 2024-11-19T17:51:02,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,689 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,689 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,689 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,689 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662675 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662682 2024-11-19T17:51:02,692 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,692 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,692 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,692 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=917, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,692 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,692 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662682) roll requested 2024-11-19T17:51:02,693 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662693 2024-11-19T17:51:02,698 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,698 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,698 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,698 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,698 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,698 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662682 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662693 2024-11-19T17:51:02,699 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,699 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,699 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,699 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=918, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,699 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,699 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662693) roll requested 2024-11-19T17:51:02,699 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662699 2024-11-19T17:51:02,704 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,704 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,704 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,704 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,704 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,704 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662693 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662699 2024-11-19T17:51:02,705 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,705 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,705 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,705 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=919, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,705 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662699) roll requested 2024-11-19T17:51:02,705 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662705 2024-11-19T17:51:02,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,710 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662699 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662705 2024-11-19T17:51:02,711 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,711 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,711 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,711 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=920, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,711 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,711 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662705) roll requested 2024-11-19T17:51:02,711 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662711 2024-11-19T17:51:02,715 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662705 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662711 2024-11-19T17:51:02,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,716 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,716 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,716 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=921, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,716 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,716 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662711) roll requested 2024-11-19T17:51:02,716 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662716 2024-11-19T17:51:02,721 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,721 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,721 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,721 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,721 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,721 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662711 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662716 2024-11-19T17:51:02,722 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,722 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,722 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,722 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=922, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,722 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,722 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662716) roll requested 2024-11-19T17:51:02,722 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662722 2024-11-19T17:51:02,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,727 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662716 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662722 2024-11-19T17:51:02,727 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,727 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,727 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,728 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=923, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,728 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,728 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662722) roll requested 2024-11-19T17:51:02,728 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662728 2024-11-19T17:51:02,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662722 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662728 2024-11-19T17:51:02,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,734 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,734 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=924, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,734 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,734 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,734 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662728) roll requested 2024-11-19T17:51:02,734 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662734 2024-11-19T17:51:02,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,738 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,738 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,738 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,738 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662728 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662734 2024-11-19T17:51:02,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,739 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,739 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,740 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=925, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,740 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,740 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662734) roll requested 2024-11-19T17:51:02,740 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662740 2024-11-19T17:51:02,745 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,745 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,745 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662734 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662740 2024-11-19T17:51:02,746 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,746 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,746 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,746 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=926, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,746 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662740) roll requested 2024-11-19T17:51:02,746 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662746 2024-11-19T17:51:02,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,751 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,751 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,751 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662740 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662746 2024-11-19T17:51:02,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,752 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,752 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=927, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,752 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662746) roll requested 2024-11-19T17:51:02,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662753 2024-11-19T17:51:02,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662746 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662753 2024-11-19T17:51:02,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,759 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,759 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=928, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,759 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662753) roll requested 2024-11-19T17:51:02,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662759 2024-11-19T17:51:02,764 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,764 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,764 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,764 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,764 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,764 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662753 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662759 2024-11-19T17:51:02,765 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,765 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,765 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,766 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=929, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,766 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,766 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662759) roll requested 2024-11-19T17:51:02,766 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662766 2024-11-19T17:51:02,770 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,770 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,770 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,770 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,771 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662759 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662766 2024-11-19T17:51:02,771 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,773 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,773 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=930, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,773 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662766) roll requested 2024-11-19T17:51:02,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662773 2024-11-19T17:51:02,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662766 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662773 2024-11-19T17:51:02,779 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,779 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,779 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,779 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=931, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,779 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662773) roll requested 2024-11-19T17:51:02,779 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662779 2024-11-19T17:51:02,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,784 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662773 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662779 2024-11-19T17:51:02,785 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,785 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,785 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,785 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=932, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,785 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662779) roll requested 2024-11-19T17:51:02,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662785 2024-11-19T17:51:02,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,791 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,791 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,791 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662779 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662785 2024-11-19T17:51:02,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,791 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,792 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,792 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=933, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,792 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,792 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662785) roll requested 2024-11-19T17:51:02,792 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662792 2024-11-19T17:51:02,796 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,796 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,797 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,797 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,797 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,797 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662785 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662792 2024-11-19T17:51:02,797 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,797 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,798 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,798 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=934, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,798 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,798 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662792) roll requested 2024-11-19T17:51:02,798 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662798 2024-11-19T17:51:02,802 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,802 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,802 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,802 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,802 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662792 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662798 2024-11-19T17:51:02,803 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,803 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,803 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,803 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=935, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,803 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,803 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662798) roll requested 2024-11-19T17:51:02,803 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662803 2024-11-19T17:51:02,808 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,809 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,809 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,809 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,809 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,809 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662798 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662803 2024-11-19T17:51:02,809 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,809 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,809 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,810 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=936, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,810 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,810 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662803) roll requested 2024-11-19T17:51:02,810 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662810 2024-11-19T17:51:02,814 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,814 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,814 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,814 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,815 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,815 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662803 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662810 2024-11-19T17:51:02,815 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,815 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,815 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,816 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=937, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,816 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,816 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662810) roll requested 2024-11-19T17:51:02,816 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662816 2024-11-19T17:51:02,821 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,821 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662810 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662816 2024-11-19T17:51:02,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,822 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,822 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=938, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,822 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,822 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662816) roll requested 2024-11-19T17:51:02,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662822 2024-11-19T17:51:02,826 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,826 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,826 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,826 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,826 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,826 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662816 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662822 2024-11-19T17:51:02,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,827 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,827 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,827 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=939, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,827 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,827 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662822) roll requested 2024-11-19T17:51:02,827 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662827 2024-11-19T17:51:02,832 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,832 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,832 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,832 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,832 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,832 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662822 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662827 2024-11-19T17:51:02,833 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,833 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,833 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,833 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=940, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,834 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,834 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662827) roll requested 2024-11-19T17:51:02,834 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662834 2024-11-19T17:51:02,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662827 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662834 2024-11-19T17:51:02,839 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,840 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,840 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=941, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,840 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662834) roll requested 2024-11-19T17:51:02,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662840 2024-11-19T17:51:02,844 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,844 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,844 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,844 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,844 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,844 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662834 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662840 2024-11-19T17:51:02,845 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,845 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,845 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,846 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=942, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,846 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,846 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662840) roll requested 2024-11-19T17:51:02,846 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662846 2024-11-19T17:51:02,851 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,851 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,851 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,851 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,851 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,851 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662840 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662846 2024-11-19T17:51:02,852 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,852 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,852 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,852 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=943, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,852 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,852 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662846) roll requested 2024-11-19T17:51:02,852 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662852 2024-11-19T17:51:02,857 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,857 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,857 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,857 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662846 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662852 2024-11-19T17:51:02,858 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,858 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,858 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,858 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=944, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,858 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,858 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662852) roll requested 2024-11-19T17:51:02,858 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662858 2024-11-19T17:51:02,863 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,863 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,863 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662852 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662858 2024-11-19T17:51:02,863 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,863 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,864 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=945, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,864 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,864 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662858) roll requested 2024-11-19T17:51:02,864 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662864 2024-11-19T17:51:02,868 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,868 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,869 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,869 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,869 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,869 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662858 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662864 2024-11-19T17:51:02,869 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,869 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,870 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,870 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=946, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,870 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,870 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662864) roll requested 2024-11-19T17:51:02,870 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662870 2024-11-19T17:51:02,875 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,875 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,875 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,875 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,875 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662864 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662870 2024-11-19T17:51:02,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,876 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,876 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=947, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,876 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662870) roll requested 2024-11-19T17:51:02,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662876 2024-11-19T17:51:02,881 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,881 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,881 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,881 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,881 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,882 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662870 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662876 2024-11-19T17:51:02,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,883 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,883 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,884 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=948, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,884 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,884 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662876) roll requested 2024-11-19T17:51:02,884 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662884 2024-11-19T17:51:02,889 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,889 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,889 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,890 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662876 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662884 2024-11-19T17:51:02,890 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,890 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,890 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=949, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,890 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,890 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662884) roll requested 2024-11-19T17:51:02,891 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662891 2024-11-19T17:51:02,895 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,895 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,895 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,895 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,895 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,896 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662884 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662891 2024-11-19T17:51:02,896 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,896 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,896 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,896 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=950, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,897 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,897 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662891) roll requested 2024-11-19T17:51:02,897 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662897 2024-11-19T17:51:02,902 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,902 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,902 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,902 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,902 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,902 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662891 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662897 2024-11-19T17:51:02,903 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,903 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,903 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,903 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=951, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,903 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,903 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662897) roll requested 2024-11-19T17:51:02,903 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662903 2024-11-19T17:51:02,908 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,908 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,908 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,908 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,908 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,908 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662897 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662903 2024-11-19T17:51:02,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,909 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,909 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,909 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=952, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,909 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662903) roll requested 2024-11-19T17:51:02,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662909 2024-11-19T17:51:02,914 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,914 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,914 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,914 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662903 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662909 2024-11-19T17:51:02,915 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,915 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,915 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,915 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=953, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,915 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,915 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662909) roll requested 2024-11-19T17:51:02,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662915 2024-11-19T17:51:02,920 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662909 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662915 2024-11-19T17:51:02,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,921 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,921 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=954, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,921 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662915) roll requested 2024-11-19T17:51:02,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662921 2024-11-19T17:51:02,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,927 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,927 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662915 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662921 2024-11-19T17:51:02,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,927 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,928 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,928 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=955, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,928 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662921) roll requested 2024-11-19T17:51:02,928 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662928 2024-11-19T17:51:02,932 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,932 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,933 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,933 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,933 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662921 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662928 2024-11-19T17:51:02,933 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,933 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,933 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,934 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=956, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,934 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,934 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662928) roll requested 2024-11-19T17:51:02,934 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662934 2024-11-19T17:51:02,938 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,938 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,938 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,938 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,938 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,938 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662928 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662934 2024-11-19T17:51:02,939 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,939 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,939 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,940 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=957, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,940 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662934) roll requested 2024-11-19T17:51:02,940 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662940 2024-11-19T17:51:02,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,945 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662934 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662940 2024-11-19T17:51:02,946 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,946 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,946 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,946 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=958, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,946 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,946 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662940) roll requested 2024-11-19T17:51:02,946 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662946 2024-11-19T17:51:02,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662940 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662946 2024-11-19T17:51:02,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,952 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,952 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=959, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,952 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662946) roll requested 2024-11-19T17:51:02,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662952 2024-11-19T17:51:02,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662946 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662952 2024-11-19T17:51:02,957 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,957 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,958 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=960, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,958 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662952) roll requested 2024-11-19T17:51:02,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662958 2024-11-19T17:51:02,962 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,962 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,962 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,962 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,962 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,963 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662952 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662958 2024-11-19T17:51:02,963 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,963 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,963 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,964 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=961, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,964 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662958) roll requested 2024-11-19T17:51:02,964 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662964 2024-11-19T17:51:02,969 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,969 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,969 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,969 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,969 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,969 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662958 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662964 2024-11-19T17:51:02,969 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,969 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,969 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,970 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=962, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,970 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,970 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662964) roll requested 2024-11-19T17:51:02,970 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662970 2024-11-19T17:51:02,974 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,974 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,974 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,974 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,975 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,975 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662964 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662970 2024-11-19T17:51:02,975 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,975 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,975 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,975 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=963, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,976 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,976 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662970) roll requested 2024-11-19T17:51:02,976 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662976 2024-11-19T17:51:02,981 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,981 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662970 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662976 2024-11-19T17:51:02,982 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,982 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,982 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,982 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=964, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,982 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,982 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662976) roll requested 2024-11-19T17:51:02,982 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662982 2024-11-19T17:51:02,987 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,987 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,987 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,987 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,987 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,987 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662976 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662982 2024-11-19T17:51:02,988 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,988 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,988 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,988 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=965, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,988 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,988 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662982) roll requested 2024-11-19T17:51:02,988 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662988 2024-11-19T17:51:02,993 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,993 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,993 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,993 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,993 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:02,993 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662982 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662988 2024-11-19T17:51:02,994 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:02,994 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,994 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:02,994 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=966, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:02,994 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:02,994 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662988) roll requested 2024-11-19T17:51:02,995 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038662995 2024-11-19T17:51:02,999 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,000 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,000 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,000 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,000 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,000 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662988 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662995 2024-11-19T17:51:03,001 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,001 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,001 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,001 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=967, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,001 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,001 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038662995) roll requested 2024-11-19T17:51:03,001 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663001 2024-11-19T17:51:03,006 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,006 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,006 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038662995 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663001 2024-11-19T17:51:03,007 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,007 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,007 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=968, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,007 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663001) roll requested 2024-11-19T17:51:03,008 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663007 2024-11-19T17:51:03,012 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,012 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,012 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,012 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,012 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,012 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663001 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663007 2024-11-19T17:51:03,013 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,013 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,013 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,013 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=969, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,013 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,013 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663007) roll requested 2024-11-19T17:51:03,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663013 2024-11-19T17:51:03,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,018 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,018 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663007 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663013 2024-11-19T17:51:03,019 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,019 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,019 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,019 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=970, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,019 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,019 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663013) roll requested 2024-11-19T17:51:03,019 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663019 2024-11-19T17:51:03,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663013 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663019 2024-11-19T17:51:03,024 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,024 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,025 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=971, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,025 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663019) roll requested 2024-11-19T17:51:03,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663025 2024-11-19T17:51:03,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,030 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,030 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663019 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663025 2024-11-19T17:51:03,031 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,031 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,031 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,031 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=972, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,031 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,031 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663025) roll requested 2024-11-19T17:51:03,031 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663031 2024-11-19T17:51:03,036 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,036 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,036 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,036 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,036 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663025 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663031 2024-11-19T17:51:03,037 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,037 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,037 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=973, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,037 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663031) roll requested 2024-11-19T17:51:03,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663037 2024-11-19T17:51:03,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,042 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,042 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,042 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,042 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,042 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663031 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663037 2024-11-19T17:51:03,043 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,043 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,043 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,043 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=974, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,043 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,043 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663037) roll requested 2024-11-19T17:51:03,043 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663043 2024-11-19T17:51:03,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,048 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,048 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,048 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,048 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,048 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663037 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663043 2024-11-19T17:51:03,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,048 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,049 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=975, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,049 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663043) roll requested 2024-11-19T17:51:03,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663049 2024-11-19T17:51:03,054 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,054 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,054 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,054 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,054 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,054 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663043 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663049 2024-11-19T17:51:03,055 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,055 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,055 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,055 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=976, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,055 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,055 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663049) roll requested 2024-11-19T17:51:03,056 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663055 2024-11-19T17:51:03,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,061 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663049 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663055 2024-11-19T17:51:03,062 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,062 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,062 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,062 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=977, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,062 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,062 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663055) roll requested 2024-11-19T17:51:03,062 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663062 2024-11-19T17:51:03,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663055 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663062 2024-11-19T17:51:03,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,070 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,071 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=978, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,071 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,071 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663062) roll requested 2024-11-19T17:51:03,071 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663071 2024-11-19T17:51:03,075 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663062 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663071 2024-11-19T17:51:03,076 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,076 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,076 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,077 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=979, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,077 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663071) roll requested 2024-11-19T17:51:03,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663077 2024-11-19T17:51:03,081 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,081 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,081 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,081 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,081 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,081 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663071 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663077 2024-11-19T17:51:03,085 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,085 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,085 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,085 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=980, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,085 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663077) roll requested 2024-11-19T17:51:03,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663085 2024-11-19T17:51:03,092 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,092 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,092 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,092 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,092 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,092 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663077 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663085 2024-11-19T17:51:03,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,096 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,096 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=981, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,096 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663085) roll requested 2024-11-19T17:51:03,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663097 2024-11-19T17:51:03,101 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,101 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,101 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,102 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,102 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,102 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663085 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663097 2024-11-19T17:51:03,104 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,104 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,105 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=982, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,105 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663097) roll requested 2024-11-19T17:51:03,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663105 2024-11-19T17:51:03,109 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,109 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,109 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,109 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,109 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663097 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663105 2024-11-19T17:51:03,110 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,110 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,110 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,110 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=983, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,111 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,111 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663105) roll requested 2024-11-19T17:51:03,111 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663111 2024-11-19T17:51:03,115 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,115 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,115 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,115 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663105 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663111 2024-11-19T17:51:03,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,116 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,116 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=984, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,116 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663111) roll requested 2024-11-19T17:51:03,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663116 2024-11-19T17:51:03,120 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,121 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,121 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,121 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,121 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,121 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663111 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663116 2024-11-19T17:51:03,122 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,122 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,122 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,122 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=985, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,123 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,123 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663116) roll requested 2024-11-19T17:51:03,123 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663123 2024-11-19T17:51:03,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,127 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,127 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,127 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663116 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663123 2024-11-19T17:51:03,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,128 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,128 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=986, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,128 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,128 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663123) roll requested 2024-11-19T17:51:03,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663128 2024-11-19T17:51:03,133 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,133 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,133 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,133 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,133 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,133 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663123 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663128 2024-11-19T17:51:03,136 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,136 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,136 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,136 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=987, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,136 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,136 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663128) roll requested 2024-11-19T17:51:03,136 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663136 2024-11-19T17:51:03,141 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,141 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,141 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,141 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,141 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,141 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663128 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663136 2024-11-19T17:51:03,142 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,142 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,142 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,142 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=988, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,142 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,142 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663136) roll requested 2024-11-19T17:51:03,142 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663142 2024-11-19T17:51:03,146 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,146 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663136 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663142 2024-11-19T17:51:03,147 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,147 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,147 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,148 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=989, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,148 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663142) roll requested 2024-11-19T17:51:03,148 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663148 2024-11-19T17:51:03,152 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,152 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,152 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663142 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663148 2024-11-19T17:51:03,153 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,153 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,153 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=990, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,153 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663148) roll requested 2024-11-19T17:51:03,154 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663153 2024-11-19T17:51:03,159 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,159 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,159 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,159 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663148 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663153 2024-11-19T17:51:03,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,160 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,160 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=991, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,160 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663153) roll requested 2024-11-19T17:51:03,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663160 2024-11-19T17:51:03,165 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,165 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,165 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,165 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,165 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,165 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663153 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663160 2024-11-19T17:51:03,166 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,166 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,166 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,166 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=992, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,166 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,166 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663160) roll requested 2024-11-19T17:51:03,166 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663166 2024-11-19T17:51:03,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663160 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663166 2024-11-19T17:51:03,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,172 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,172 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=993, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,172 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663166) roll requested 2024-11-19T17:51:03,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663173 2024-11-19T17:51:03,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,177 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,177 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,178 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,178 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,178 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663166 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663173 2024-11-19T17:51:03,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,178 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,179 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=994, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,179 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663173) roll requested 2024-11-19T17:51:03,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663179 2024-11-19T17:51:03,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663173 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663179 2024-11-19T17:51:03,185 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,185 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,185 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,185 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=995, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,185 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663179) roll requested 2024-11-19T17:51:03,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663185 2024-11-19T17:51:03,189 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,189 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,189 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,190 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,190 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,190 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663179 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663185 2024-11-19T17:51:03,190 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,190 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,190 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,191 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=996, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,191 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,191 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663185) roll requested 2024-11-19T17:51:03,191 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663191 2024-11-19T17:51:03,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,197 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663185 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663191 2024-11-19T17:51:03,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,198 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,198 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,198 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=997, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,198 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,198 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663191) roll requested 2024-11-19T17:51:03,198 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663198 2024-11-19T17:51:03,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663191 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663198 2024-11-19T17:51:03,205 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,205 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,205 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,205 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=998, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,206 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,206 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663198) roll requested 2024-11-19T17:51:03,206 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663206 2024-11-19T17:51:03,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663198 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663206 2024-11-19T17:51:03,211 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,211 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,211 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=999, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,211 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,211 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663206) roll requested 2024-11-19T17:51:03,211 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663211 2024-11-19T17:51:03,216 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,216 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,216 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,216 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,216 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,216 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663206 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663211 2024-11-19T17:51:03,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,217 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,217 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,217 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1000, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,217 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663211) roll requested 2024-11-19T17:51:03,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663217 2024-11-19T17:51:03,222 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,222 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,222 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,222 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663211 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663217 2024-11-19T17:51:03,223 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,223 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,223 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,223 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1001, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,223 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,223 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663217) roll requested 2024-11-19T17:51:03,223 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663223 2024-11-19T17:51:03,227 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663217 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663223 2024-11-19T17:51:03,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,229 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,229 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1002, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,229 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663223) roll requested 2024-11-19T17:51:03,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663229 2024-11-19T17:51:03,234 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,235 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,235 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,235 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,235 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,235 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663223 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663229 2024-11-19T17:51:03,235 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,235 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,236 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,236 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1003, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,236 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,236 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663229) roll requested 2024-11-19T17:51:03,236 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663236 2024-11-19T17:51:03,241 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,241 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,241 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,241 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,241 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,241 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663229 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663236 2024-11-19T17:51:03,241 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,242 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,242 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,242 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1004, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,242 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,242 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663236) roll requested 2024-11-19T17:51:03,242 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663242 2024-11-19T17:51:03,247 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,247 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,247 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,247 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,247 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,247 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663236 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663242 2024-11-19T17:51:03,248 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,248 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,248 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1005, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,248 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,248 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,248 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663242) roll requested 2024-11-19T17:51:03,248 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663248 2024-11-19T17:51:03,253 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,253 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,253 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,253 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,253 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,254 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663242 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663248 2024-11-19T17:51:03,254 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,254 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,254 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,254 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1006, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,254 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,254 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663248) roll requested 2024-11-19T17:51:03,254 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663254 2024-11-19T17:51:03,259 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,259 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,259 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,259 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,259 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,259 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663248 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663254 2024-11-19T17:51:03,260 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,260 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,260 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,260 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1007, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,260 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,260 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663254) roll requested 2024-11-19T17:51:03,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663260 2024-11-19T17:51:03,265 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,265 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,265 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,265 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,266 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663254 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663260 2024-11-19T17:51:03,266 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,266 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,266 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,267 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1008, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,267 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,267 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663260) roll requested 2024-11-19T17:51:03,267 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663267 2024-11-19T17:51:03,271 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,271 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,271 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,271 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663260 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663267 2024-11-19T17:51:03,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,272 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,272 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1009, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,272 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,272 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663267) roll requested 2024-11-19T17:51:03,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663272 2024-11-19T17:51:03,277 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,277 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,277 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663267 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663272 2024-11-19T17:51:03,278 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,278 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,278 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,278 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1010, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,279 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,279 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663272) roll requested 2024-11-19T17:51:03,279 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663279 2024-11-19T17:51:03,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,283 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,283 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,283 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,283 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663272 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663279 2024-11-19T17:51:03,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,284 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,285 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1011, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,285 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663279) roll requested 2024-11-19T17:51:03,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663285 2024-11-19T17:51:03,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,290 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,290 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,290 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,290 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,290 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663279 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663285 2024-11-19T17:51:03,290 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,290 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,291 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,291 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1012, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,291 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,291 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663285) roll requested 2024-11-19T17:51:03,291 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663291 2024-11-19T17:51:03,296 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,296 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,296 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,297 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,297 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,297 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663285 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663291 2024-11-19T17:51:03,297 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,297 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,297 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,298 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1013, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,298 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,298 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663291) roll requested 2024-11-19T17:51:03,298 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663298 2024-11-19T17:51:03,302 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,302 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,302 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,302 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,302 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,303 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663291 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663298 2024-11-19T17:51:03,303 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,303 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,303 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,303 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1014, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,303 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,303 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663298) roll requested 2024-11-19T17:51:03,304 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663303 2024-11-19T17:51:03,308 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,308 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,308 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,308 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,308 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,308 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663298 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663303 2024-11-19T17:51:03,309 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,309 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,309 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,309 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1015, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,309 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,309 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663303) roll requested 2024-11-19T17:51:03,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663309 2024-11-19T17:51:03,314 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,314 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,314 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,314 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,315 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,315 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663303 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663309 2024-11-19T17:51:03,315 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,315 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,315 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,316 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1016, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,316 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,316 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663309) roll requested 2024-11-19T17:51:03,316 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663316 2024-11-19T17:51:03,320 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,320 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,320 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,321 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,321 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,321 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663309 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663316 2024-11-19T17:51:03,321 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,321 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,322 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,322 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1017, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,322 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,322 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663316) roll requested 2024-11-19T17:51:03,322 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663322 2024-11-19T17:51:03,326 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,326 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,327 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,327 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,327 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,327 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663316 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663322 2024-11-19T17:51:03,329 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,329 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,329 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,329 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1018, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,329 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663322) roll requested 2024-11-19T17:51:03,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663329 2024-11-19T17:51:03,334 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,334 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,334 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,335 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,335 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,335 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663322 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663329 2024-11-19T17:51:03,335 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,335 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,335 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,336 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1019, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,336 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663329) roll requested 2024-11-19T17:51:03,336 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663336 2024-11-19T17:51:03,341 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,341 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,341 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,341 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,341 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,341 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663329 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663336 2024-11-19T17:51:03,341 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,341 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,341 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,342 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1020, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,342 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,342 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663336) roll requested 2024-11-19T17:51:03,342 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663342 2024-11-19T17:51:03,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,346 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,346 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,347 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,347 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663336 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663342 2024-11-19T17:51:03,347 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,347 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,347 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,348 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1021, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,348 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,348 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663342) roll requested 2024-11-19T17:51:03,348 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663348 2024-11-19T17:51:03,352 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,352 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,352 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,352 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,352 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663342 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663348 2024-11-19T17:51:03,353 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,353 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,353 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,353 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1022, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,353 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,354 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663348) roll requested 2024-11-19T17:51:03,354 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663354 2024-11-19T17:51:03,358 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,358 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,358 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,358 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,358 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,358 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663348 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663354 2024-11-19T17:51:03,359 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,359 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,359 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,359 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1023, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,359 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,359 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663354) roll requested 2024-11-19T17:51:03,360 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663360 2024-11-19T17:51:03,364 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,364 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,365 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,365 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663354 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663360 2024-11-19T17:51:03,365 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,365 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,366 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,366 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1024, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,366 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,366 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663360) roll requested 2024-11-19T17:51:03,366 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663366 2024-11-19T17:51:03,370 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,370 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,370 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,370 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,371 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663360 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663366 2024-11-19T17:51:03,371 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,371 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,371 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1025, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,371 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663366) roll requested 2024-11-19T17:51:03,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663371 2024-11-19T17:51:03,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,376 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,376 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663366 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663371 2024-11-19T17:51:03,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,377 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,377 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1026, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,377 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,378 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663371) roll requested 2024-11-19T17:51:03,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663378 2024-11-19T17:51:03,382 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,382 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,383 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,383 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663371 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663378 2024-11-19T17:51:03,383 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,383 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,383 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,383 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1027, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,384 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663378) roll requested 2024-11-19T17:51:03,384 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663384 2024-11-19T17:51:03,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663378 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663384 2024-11-19T17:51:03,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,389 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,390 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1028, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,390 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,390 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663384) roll requested 2024-11-19T17:51:03,390 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663390 2024-11-19T17:51:03,394 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,394 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,394 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,394 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,394 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,395 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663384 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663390 2024-11-19T17:51:03,395 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,395 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,395 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,395 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1029, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,396 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,396 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663390) roll requested 2024-11-19T17:51:03,396 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663396 2024-11-19T17:51:03,400 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,400 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,400 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,400 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,400 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,401 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663390 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663396 2024-11-19T17:51:03,401 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,401 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,401 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,401 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1030, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,402 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,402 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663396) roll requested 2024-11-19T17:51:03,402 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663402 2024-11-19T17:51:03,406 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,406 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663396 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663402 2024-11-19T17:51:03,407 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,407 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,407 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,407 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1031, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,407 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663402) roll requested 2024-11-19T17:51:03,408 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663407 2024-11-19T17:51:03,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,413 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663402 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663407 2024-11-19T17:51:03,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,413 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,414 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,414 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1032, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,414 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,414 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663407) roll requested 2024-11-19T17:51:03,414 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663414 2024-11-19T17:51:03,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663407 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663414 2024-11-19T17:51:03,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,420 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,420 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1033, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,420 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663414) roll requested 2024-11-19T17:51:03,420 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663420 2024-11-19T17:51:03,425 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,425 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,425 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,425 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,425 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,425 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663414 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663420 2024-11-19T17:51:03,426 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,426 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,426 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,426 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1034, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,426 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663420) roll requested 2024-11-19T17:51:03,427 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663427 2024-11-19T17:51:03,431 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,432 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,432 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,432 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,432 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,432 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663420 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663427 2024-11-19T17:51:03,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,432 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,433 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1035, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,433 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663427) roll requested 2024-11-19T17:51:03,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663433 2024-11-19T17:51:03,437 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,437 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,437 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,437 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,437 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,437 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663427 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663433 2024-11-19T17:51:03,438 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,438 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,438 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,438 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1036, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,438 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,438 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663433) roll requested 2024-11-19T17:51:03,438 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663438 2024-11-19T17:51:03,442 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,442 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,442 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,442 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,442 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,442 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663433 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663438 2024-11-19T17:51:03,443 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,443 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,443 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,443 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1037, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,443 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,443 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663438) roll requested 2024-11-19T17:51:03,443 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663443 2024-11-19T17:51:03,448 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,448 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,448 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,448 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,448 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,448 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663438 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663443 2024-11-19T17:51:03,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,449 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,450 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1038, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,450 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663443) roll requested 2024-11-19T17:51:03,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663450 2024-11-19T17:51:03,454 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,455 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,455 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,455 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,455 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663443 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663450 2024-11-19T17:51:03,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,456 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,456 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1039, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,456 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663450) roll requested 2024-11-19T17:51:03,456 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663456 2024-11-19T17:51:03,462 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,462 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,462 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,462 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,462 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,462 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663450 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663456 2024-11-19T17:51:03,463 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,463 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,463 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,463 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1040, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,463 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663456) roll requested 2024-11-19T17:51:03,464 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663464 2024-11-19T17:51:03,469 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,469 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,469 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,469 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,469 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,469 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663456 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663464 2024-11-19T17:51:03,470 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,470 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,470 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,470 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1041, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,470 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,471 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663464) roll requested 2024-11-19T17:51:03,471 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663471 2024-11-19T17:51:03,476 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,476 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,476 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,476 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,476 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,476 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663464 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663471 2024-11-19T17:51:03,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,477 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,478 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1042, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,478 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663471) roll requested 2024-11-19T17:51:03,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663478 2024-11-19T17:51:03,484 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,484 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,484 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,485 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,485 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,485 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663471 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663478 2024-11-19T17:51:03,486 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,486 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,486 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,486 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1043, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,486 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,486 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663478) roll requested 2024-11-19T17:51:03,486 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663486 2024-11-19T17:51:03,492 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,493 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,493 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,493 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,493 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,493 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663478 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663486 2024-11-19T17:51:03,494 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,494 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,494 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,495 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1044, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,495 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,495 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663486) roll requested 2024-11-19T17:51:03,495 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663495 2024-11-19T17:51:03,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663486 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663495 2024-11-19T17:51:03,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,504 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,504 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1045, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,505 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663495) roll requested 2024-11-19T17:51:03,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663505 2024-11-19T17:51:03,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663495 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663505 2024-11-19T17:51:03,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,512 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,513 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,513 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1046, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,513 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,513 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663505) roll requested 2024-11-19T17:51:03,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663513 2024-11-19T17:51:03,519 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,519 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,519 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,519 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,519 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,519 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663505 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663513 2024-11-19T17:51:03,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,520 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,520 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1047, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,520 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,520 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663513) roll requested 2024-11-19T17:51:03,520 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663520 2024-11-19T17:51:03,526 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663513 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663520 2024-11-19T17:51:03,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,527 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,528 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1048, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,528 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663520) roll requested 2024-11-19T17:51:03,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663528 2024-11-19T17:51:03,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663520 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663528 2024-11-19T17:51:03,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,535 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,535 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1049, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,536 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,536 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663528) roll requested 2024-11-19T17:51:03,536 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663536 2024-11-19T17:51:03,543 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,543 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663528 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663536 2024-11-19T17:51:03,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,544 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,545 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1050, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,545 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,545 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663536) roll requested 2024-11-19T17:51:03,546 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663546 2024-11-19T17:51:03,553 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,553 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,553 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,553 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,554 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,554 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663536 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663546 2024-11-19T17:51:03,555 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,555 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,555 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,555 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1051, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,555 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,555 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663546) roll requested 2024-11-19T17:51:03,555 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663555 2024-11-19T17:51:03,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,564 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,564 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,564 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,564 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663546 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663555 2024-11-19T17:51:03,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,564 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,565 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1052, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,565 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663555) roll requested 2024-11-19T17:51:03,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663565 2024-11-19T17:51:03,569 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,569 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,569 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663555 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663565 2024-11-19T17:51:03,570 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,570 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,570 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,570 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1053, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,571 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663565) roll requested 2024-11-19T17:51:03,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663571 2024-11-19T17:51:03,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663565 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663571 2024-11-19T17:51:03,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,576 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,577 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1054, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,577 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663571) roll requested 2024-11-19T17:51:03,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663577 2024-11-19T17:51:03,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,584 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663571 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663577 2024-11-19T17:51:03,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,584 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,585 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1055, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,585 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663577) roll requested 2024-11-19T17:51:03,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663585 2024-11-19T17:51:03,591 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,591 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,591 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663577 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663585 2024-11-19T17:51:03,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,592 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,592 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1056, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,592 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663585) roll requested 2024-11-19T17:51:03,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663592 2024-11-19T17:51:03,597 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,597 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,597 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,597 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,597 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663585 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663592 2024-11-19T17:51:03,598 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,598 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,598 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,599 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1057, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,599 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663592) roll requested 2024-11-19T17:51:03,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663599 2024-11-19T17:51:03,604 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,604 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,604 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,604 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,604 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,604 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663592 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663599 2024-11-19T17:51:03,605 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,605 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,605 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,605 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1058, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,605 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,605 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663599) roll requested 2024-11-19T17:51:03,605 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663605 2024-11-19T17:51:03,610 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,611 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,611 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,611 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,611 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,611 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663599 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663605 2024-11-19T17:51:03,611 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,612 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,612 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1059, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,612 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663605) roll requested 2024-11-19T17:51:03,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663612 2024-11-19T17:51:03,617 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,617 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663605 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663612 2024-11-19T17:51:03,618 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,618 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,618 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,618 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1060, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,618 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,618 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663612) roll requested 2024-11-19T17:51:03,619 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663618 2024-11-19T17:51:03,623 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,624 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,624 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,624 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,624 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663612 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663618 2024-11-19T17:51:03,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,624 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,625 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1061, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,625 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663618) roll requested 2024-11-19T17:51:03,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663625 2024-11-19T17:51:03,629 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,630 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,630 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,630 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663618 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663625 2024-11-19T17:51:03,630 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,630 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,631 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,631 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1062, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,631 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,631 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663625) roll requested 2024-11-19T17:51:03,631 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663631 2024-11-19T17:51:03,636 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,636 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,636 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,636 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,636 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,636 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663625 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663631 2024-11-19T17:51:03,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,637 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,637 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1063, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,637 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663631) roll requested 2024-11-19T17:51:03,637 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663637 2024-11-19T17:51:03,643 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,643 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,643 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,643 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,643 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663631 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663637 2024-11-19T17:51:03,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,644 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,644 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1064, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,644 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,644 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663637) roll requested 2024-11-19T17:51:03,644 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663644 2024-11-19T17:51:03,649 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,649 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,649 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,649 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,649 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,649 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663637 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663644 2024-11-19T17:51:03,650 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,650 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,650 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,650 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1065, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,650 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,650 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663644) roll requested 2024-11-19T17:51:03,651 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663651 2024-11-19T17:51:03,656 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,656 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,656 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,656 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,656 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,656 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663644 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663651 2024-11-19T17:51:03,657 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,657 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,657 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,657 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1066, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,657 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,657 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663651) roll requested 2024-11-19T17:51:03,657 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663657 2024-11-19T17:51:03,662 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,662 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,662 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,662 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,662 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,662 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663651 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663657 2024-11-19T17:51:03,663 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,663 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,663 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,663 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1067, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,663 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,663 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663657) roll requested 2024-11-19T17:51:03,663 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663663 2024-11-19T17:51:03,668 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,668 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,668 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,668 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,668 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,669 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663657 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663663 2024-11-19T17:51:03,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,669 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,669 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1068, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,670 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,670 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663663) roll requested 2024-11-19T17:51:03,670 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663670 2024-11-19T17:51:03,674 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,674 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,674 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,674 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,675 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663663 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663670 2024-11-19T17:51:03,675 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,675 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,675 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,676 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1069, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,676 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663670) roll requested 2024-11-19T17:51:03,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663676 2024-11-19T17:51:03,681 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,681 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,681 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,681 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,681 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,681 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663670 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663676 2024-11-19T17:51:03,682 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,682 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,682 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,682 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1070, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,682 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,682 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663676) roll requested 2024-11-19T17:51:03,682 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663682 2024-11-19T17:51:03,686 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,687 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,687 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,687 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,687 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,687 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663676 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663682 2024-11-19T17:51:03,687 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,687 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,687 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,688 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1071, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,688 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,688 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663682) roll requested 2024-11-19T17:51:03,688 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663688 2024-11-19T17:51:03,693 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,693 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,693 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,693 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,693 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,693 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663682 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663688 2024-11-19T17:51:03,694 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,694 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,694 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,694 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1072, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,694 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,694 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663688) roll requested 2024-11-19T17:51:03,694 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663694 2024-11-19T17:51:03,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,700 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,700 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,700 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,700 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663688 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663694 2024-11-19T17:51:03,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,701 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,702 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1073, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,702 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,702 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663694) roll requested 2024-11-19T17:51:03,702 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663702 2024-11-19T17:51:03,706 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663694 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663702 2024-11-19T17:51:03,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,707 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,707 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,708 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1074, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,708 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663702) roll requested 2024-11-19T17:51:03,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663708 2024-11-19T17:51:03,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,713 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,713 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663702 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663708 2024-11-19T17:51:03,714 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,714 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,714 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,714 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1075, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,714 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663708) roll requested 2024-11-19T17:51:03,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663714 2024-11-19T17:51:03,719 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,719 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,720 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,720 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,720 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,720 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663708 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663714 2024-11-19T17:51:03,720 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,720 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,720 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,720 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1076, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,721 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663714) roll requested 2024-11-19T17:51:03,721 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663721 2024-11-19T17:51:03,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,726 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,726 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663714 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663721 2024-11-19T17:51:03,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,726 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,727 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1077, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,727 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,727 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663721) roll requested 2024-11-19T17:51:03,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663727 2024-11-19T17:51:03,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,732 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,732 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,732 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,732 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663721 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663727 2024-11-19T17:51:03,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,732 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,732 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,733 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1078, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,733 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663727) roll requested 2024-11-19T17:51:03,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663733 2024-11-19T17:51:03,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,738 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,738 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,738 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,738 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663727 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663733 2024-11-19T17:51:03,738 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,738 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,738 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,739 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1079, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,739 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,739 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663733) roll requested 2024-11-19T17:51:03,739 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663739 2024-11-19T17:51:03,743 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,743 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,743 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,744 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,744 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,744 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663733 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663739 2024-11-19T17:51:03,745 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,745 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,745 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,745 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1080, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,746 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663739) roll requested 2024-11-19T17:51:03,746 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663746 2024-11-19T17:51:03,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,751 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,751 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,751 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663739 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663746 2024-11-19T17:51:03,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,752 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,752 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1081, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,752 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663746) roll requested 2024-11-19T17:51:03,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663752 2024-11-19T17:51:03,757 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663746 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663752 2024-11-19T17:51:03,758 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,758 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,758 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,759 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1082, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,759 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663752) roll requested 2024-11-19T17:51:03,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663759 2024-11-19T17:51:03,764 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,764 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,764 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,764 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,764 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,764 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663752 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663759 2024-11-19T17:51:03,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,764 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,765 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1083, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,765 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,765 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663759) roll requested 2024-11-19T17:51:03,765 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663765 2024-11-19T17:51:03,770 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,770 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,770 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,771 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663759 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663765 2024-11-19T17:51:03,771 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,771 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,771 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,772 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1084, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,772 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663765) roll requested 2024-11-19T17:51:03,772 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663772 2024-11-19T17:51:03,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,777 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,777 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,777 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663765 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663772 2024-11-19T17:51:03,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,778 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,779 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1085, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,779 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663772) roll requested 2024-11-19T17:51:03,779 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663779 2024-11-19T17:51:03,783 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,783 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,784 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663772 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663779 2024-11-19T17:51:03,785 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,785 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,785 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,785 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1086, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,785 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663779) roll requested 2024-11-19T17:51:03,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663785 2024-11-19T17:51:03,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,790 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663779 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663785 2024-11-19T17:51:03,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,791 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,792 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1087, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,792 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,792 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663785) roll requested 2024-11-19T17:51:03,792 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663792 2024-11-19T17:51:03,797 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,797 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,797 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,797 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,797 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,797 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663785 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663792 2024-11-19T17:51:03,798 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,798 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,798 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,798 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1088, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,798 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,798 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663792) roll requested 2024-11-19T17:51:03,798 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663798 2024-11-19T17:51:03,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,804 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,804 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,804 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,804 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663792 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663798 2024-11-19T17:51:03,805 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,805 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,805 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,805 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1089, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,805 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663798) roll requested 2024-11-19T17:51:03,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663805 2024-11-19T17:51:03,810 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,810 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,810 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,810 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,810 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,810 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663798 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663805 2024-11-19T17:51:03,811 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,811 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,811 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,811 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1090, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,811 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,811 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663805) roll requested 2024-11-19T17:51:03,812 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663811 2024-11-19T17:51:03,816 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,816 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,816 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,816 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,816 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,816 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663805 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663811 2024-11-19T17:51:03,817 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,817 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,817 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,817 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1091, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,817 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,817 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663811) roll requested 2024-11-19T17:51:03,818 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663817 2024-11-19T17:51:03,822 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,822 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,822 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,822 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,822 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,822 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663811 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663817 2024-11-19T17:51:03,823 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,823 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,823 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,823 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1092, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,823 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,823 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663817) roll requested 2024-11-19T17:51:03,824 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663824 2024-11-19T17:51:03,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663817 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663824 2024-11-19T17:51:03,830 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,830 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,830 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,830 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1093, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,830 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,830 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663824) roll requested 2024-11-19T17:51:03,830 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663830 2024-11-19T17:51:03,834 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,834 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,834 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,834 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,834 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663824 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663830 2024-11-19T17:51:03,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,836 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,836 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1094, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,836 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663830) roll requested 2024-11-19T17:51:03,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663837 2024-11-19T17:51:03,842 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,842 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,842 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,842 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,842 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,843 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663830 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663837 2024-11-19T17:51:03,843 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,843 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,843 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,843 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1095, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,844 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,844 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663837) roll requested 2024-11-19T17:51:03,844 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663844 2024-11-19T17:51:03,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,848 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,848 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663837 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663844 2024-11-19T17:51:03,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,849 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,849 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1096, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,849 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,849 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663844) roll requested 2024-11-19T17:51:03,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663849 2024-11-19T17:51:03,854 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,854 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,854 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,854 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,854 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,854 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663844 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663849 2024-11-19T17:51:03,855 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,855 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,855 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,856 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1097, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,856 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,856 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663849) roll requested 2024-11-19T17:51:03,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663856 2024-11-19T17:51:03,861 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,861 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,862 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,862 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,862 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,862 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663849 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663856 2024-11-19T17:51:03,862 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,862 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,862 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,863 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1098, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,863 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,863 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663856) roll requested 2024-11-19T17:51:03,863 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663863 2024-11-19T17:51:03,868 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,868 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,868 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,869 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,869 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,869 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663856 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663863 2024-11-19T17:51:03,870 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,870 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,870 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,870 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1099, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,870 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,870 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663863) roll requested 2024-11-19T17:51:03,870 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663870 2024-11-19T17:51:03,876 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,876 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,876 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,876 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,876 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663863 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663870 2024-11-19T17:51:03,877 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,878 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,878 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1100, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,878 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663870) roll requested 2024-11-19T17:51:03,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663878 2024-11-19T17:51:03,885 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,885 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,886 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663870 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663878 2024-11-19T17:51:03,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,887 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,887 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1101, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,888 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663878) roll requested 2024-11-19T17:51:03,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663888 2024-11-19T17:51:03,895 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,895 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,895 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,895 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,895 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,896 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663878 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663888 2024-11-19T17:51:03,896 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,896 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,896 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,897 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1102, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,897 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,897 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663888) roll requested 2024-11-19T17:51:03,897 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663897 2024-11-19T17:51:03,902 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,902 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,902 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,902 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,902 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,902 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663888 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663897 2024-11-19T17:51:03,903 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,903 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,903 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,905 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1103, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,905 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,905 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663897) roll requested 2024-11-19T17:51:03,905 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663905 2024-11-19T17:51:03,912 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,912 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,912 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,912 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,912 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,912 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663897 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663905 2024-11-19T17:51:03,913 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,913 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,913 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,914 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1104, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,914 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,914 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663905) roll requested 2024-11-19T17:51:03,914 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663914 2024-11-19T17:51:03,920 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663905 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663914 2024-11-19T17:51:03,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,921 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,922 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,922 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1105, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,922 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,922 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663914) roll requested 2024-11-19T17:51:03,922 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663922 2024-11-19T17:51:03,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,932 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663914 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663922 2024-11-19T17:51:03,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,932 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,933 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,933 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1106, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,933 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663922) roll requested 2024-11-19T17:51:03,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663933 2024-11-19T17:51:03,940 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,940 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,940 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,940 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,941 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,941 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663922 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663933 2024-11-19T17:51:03,941 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,941 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,941 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,942 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1107, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,942 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,942 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663933) roll requested 2024-11-19T17:51:03,942 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663942 2024-11-19T17:51:03,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,946 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,946 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663933 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663942 2024-11-19T17:51:03,947 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,947 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,947 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,947 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1108, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,947 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,947 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663942) roll requested 2024-11-19T17:51:03,948 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663948 2024-11-19T17:51:03,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,952 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,952 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,952 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,952 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,952 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663942 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663948 2024-11-19T17:51:03,953 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,953 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,953 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,953 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1109, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,953 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663948) roll requested 2024-11-19T17:51:03,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663953 2024-11-19T17:51:03,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,958 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,958 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,958 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,958 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,958 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663948 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663953 2024-11-19T17:51:03,959 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,959 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,959 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,960 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1110, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,960 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663953) roll requested 2024-11-19T17:51:03,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663960 2024-11-19T17:51:03,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,966 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,966 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663953 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663960 2024-11-19T17:51:03,967 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,967 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,968 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1111, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,968 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,968 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663960) roll requested 2024-11-19T17:51:03,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663968 2024-11-19T17:51:03,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,976 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663960 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663968 2024-11-19T17:51:03,977 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,977 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,978 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,978 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1112, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,978 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,978 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663968) roll requested 2024-11-19T17:51:03,978 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663978 2024-11-19T17:51:03,987 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,988 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,988 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,988 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,988 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663968 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663978 2024-11-19T17:51:03,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,989 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,989 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,989 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1113, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,989 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,989 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663978) roll requested 2024-11-19T17:51:03,989 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663989 2024-11-19T17:51:03,993 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,994 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,994 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,994 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,994 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,994 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663978 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663989 2024-11-19T17:51:03,994 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:03,994 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,994 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:03,995 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1114, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:03,995 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:03,995 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663989) roll requested 2024-11-19T17:51:03,995 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038663995 2024-11-19T17:51:03,999 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,999 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,999 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,999 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:03,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663989 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663995 2024-11-19T17:51:04,000 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,000 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,000 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1115, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,001 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,001 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038663995) roll requested 2024-11-19T17:51:04,001 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664001 2024-11-19T17:51:04,005 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,005 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,005 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038663995 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664001 2024-11-19T17:51:04,006 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,006 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,006 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,007 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1116, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,007 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664001) roll requested 2024-11-19T17:51:04,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664007 2024-11-19T17:51:04,011 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,011 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,012 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,012 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,012 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664001 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664007 2024-11-19T17:51:04,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,012 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,013 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,013 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1117, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,013 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,013 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664007) roll requested 2024-11-19T17:51:04,013 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664013 2024-11-19T17:51:04,017 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,017 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,018 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,018 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664007 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664013 2024-11-19T17:51:04,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,018 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,019 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,019 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1118, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,019 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,019 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664013) roll requested 2024-11-19T17:51:04,019 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664019 2024-11-19T17:51:04,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664013 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664019 2024-11-19T17:51:04,024 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,024 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,025 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1119, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,025 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664019) roll requested 2024-11-19T17:51:04,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664025 2024-11-19T17:51:04,029 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,029 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,029 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,029 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,029 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664019 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664025 2024-11-19T17:51:04,030 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,030 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,030 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,031 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1120, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,031 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,031 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664025) roll requested 2024-11-19T17:51:04,031 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664031 2024-11-19T17:51:04,035 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,035 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,035 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,035 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664025 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664031 2024-11-19T17:51:04,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,036 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,036 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,036 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1121, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,036 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,036 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664031) roll requested 2024-11-19T17:51:04,036 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664036 2024-11-19T17:51:04,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,042 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,042 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,042 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,042 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,042 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664031 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664036 2024-11-19T17:51:04,043 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,043 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,043 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,043 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1122, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,043 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,043 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664036) roll requested 2024-11-19T17:51:04,043 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664043 2024-11-19T17:51:04,048 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,048 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,048 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,048 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,048 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,048 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664036 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664043 2024-11-19T17:51:04,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,048 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,049 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1123, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,049 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664043) roll requested 2024-11-19T17:51:04,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664049 2024-11-19T17:51:04,054 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,054 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,054 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,054 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,054 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,054 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664043 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664049 2024-11-19T17:51:04,055 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,055 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,055 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,055 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1124, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,055 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,055 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664049) roll requested 2024-11-19T17:51:04,055 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664055 2024-11-19T17:51:04,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,060 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,060 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,060 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,060 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,060 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664049 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664055 2024-11-19T17:51:04,060 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,060 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,061 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,061 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1125, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,061 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,061 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664055) roll requested 2024-11-19T17:51:04,061 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664061 2024-11-19T17:51:04,066 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,066 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,066 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,066 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,066 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,066 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664055 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664061 2024-11-19T17:51:04,067 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,067 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,067 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,067 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1126, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,067 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,067 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664061) roll requested 2024-11-19T17:51:04,067 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664067 2024-11-19T17:51:04,072 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,072 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,072 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,073 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,073 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,073 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664061 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664067 2024-11-19T17:51:04,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,073 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,074 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1127, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,074 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,074 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664067) roll requested 2024-11-19T17:51:04,074 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664074 2024-11-19T17:51:04,078 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,078 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,078 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,079 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,079 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664067 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664074 2024-11-19T17:51:04,079 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,079 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,079 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,080 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1128, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,080 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,080 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664074) roll requested 2024-11-19T17:51:04,080 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664080 2024-11-19T17:51:04,085 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,085 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,085 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,085 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,085 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664074 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664080 2024-11-19T17:51:04,086 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,086 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,086 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,086 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1129, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,086 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,086 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664080) roll requested 2024-11-19T17:51:04,086 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664086 2024-11-19T17:51:04,090 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,090 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,091 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,091 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664080 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664086 2024-11-19T17:51:04,091 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,091 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,091 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,092 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1130, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,092 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,092 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664086) roll requested 2024-11-19T17:51:04,092 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664092 2024-11-19T17:51:04,096 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,096 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,096 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,096 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,097 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,097 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664086 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664092 2024-11-19T17:51:04,097 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,097 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,098 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,098 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1131, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,098 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,098 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664092) roll requested 2024-11-19T17:51:04,098 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664098 2024-11-19T17:51:04,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664092 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664098 2024-11-19T17:51:04,104 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,104 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,104 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1132, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,105 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664098) roll requested 2024-11-19T17:51:04,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664105 2024-11-19T17:51:04,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,113 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664098 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664105 2024-11-19T17:51:04,114 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,114 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,114 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,114 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1133, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,114 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,114 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664105) roll requested 2024-11-19T17:51:04,115 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664115 2024-11-19T17:51:04,123 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,124 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,124 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,124 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664105 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664115 2024-11-19T17:51:04,125 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,125 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,125 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,125 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1134, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,125 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,125 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664115) roll requested 2024-11-19T17:51:04,125 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664125 2024-11-19T17:51:04,130 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,130 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,130 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,130 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,130 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,130 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664115 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664125 2024-11-19T17:51:04,135 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,135 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,135 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,135 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1135, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,135 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,135 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664125) roll requested 2024-11-19T17:51:04,135 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664135 2024-11-19T17:51:04,140 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,141 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,141 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,141 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,145 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,145 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664125 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664135 2024-11-19T17:51:04,147 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,147 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,147 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,148 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1136, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,148 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664135) roll requested 2024-11-19T17:51:04,148 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664148 2024-11-19T17:51:04,152 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,152 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,152 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,153 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664135 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664148 2024-11-19T17:51:04,153 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,153 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,154 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1137, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,154 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,154 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664148) roll requested 2024-11-19T17:51:04,154 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664154 2024-11-19T17:51:04,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,159 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664148 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664154 2024-11-19T17:51:04,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,159 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,160 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1138, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,160 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664154) roll requested 2024-11-19T17:51:04,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664160 2024-11-19T17:51:04,165 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,165 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,165 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,165 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,165 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,165 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664154 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664160 2024-11-19T17:51:04,166 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,166 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,166 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,166 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1139, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,166 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,166 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664160) roll requested 2024-11-19T17:51:04,166 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664166 2024-11-19T17:51:04,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664160 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664166 2024-11-19T17:51:04,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,172 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,172 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1140, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,172 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664166) roll requested 2024-11-19T17:51:04,172 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664172 2024-11-19T17:51:04,176 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,176 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,176 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,177 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,177 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,177 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664166 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664172 2024-11-19T17:51:04,177 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,177 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,178 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,178 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1141, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,178 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,178 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664172) roll requested 2024-11-19T17:51:04,178 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664178 2024-11-19T17:51:04,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,182 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,182 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664172 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664178 2024-11-19T17:51:04,183 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,183 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,183 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,183 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1142, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,183 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,183 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664178) roll requested 2024-11-19T17:51:04,183 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664183 2024-11-19T17:51:04,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,188 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664178 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664183 2024-11-19T17:51:04,189 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,189 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,190 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1143, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,190 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,190 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664183) roll requested 2024-11-19T17:51:04,190 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664190 2024-11-19T17:51:04,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664183 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664190 2024-11-19T17:51:04,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,209 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1144, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,209 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,209 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,209 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,209 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664190) roll requested 2024-11-19T17:51:04,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664209 2024-11-19T17:51:04,213 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,214 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,214 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,214 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,214 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,214 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664190 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664209 2024-11-19T17:51:04,214 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,214 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,214 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,215 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1145, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,215 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,215 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664209) roll requested 2024-11-19T17:51:04,215 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664215 2024-11-19T17:51:04,219 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,219 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,219 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,219 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664209 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664215 2024-11-19T17:51:04,220 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,220 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,220 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,220 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1146, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,220 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,220 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664215) roll requested 2024-11-19T17:51:04,220 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664220 2024-11-19T17:51:04,225 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,225 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,225 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664215 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664220 2024-11-19T17:51:04,226 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,226 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,226 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,226 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1147, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,226 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664220) roll requested 2024-11-19T17:51:04,227 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664226 2024-11-19T17:51:04,231 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,231 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,231 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,231 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,232 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,232 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664220 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664226 2024-11-19T17:51:04,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,232 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,233 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1148, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,233 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664226) roll requested 2024-11-19T17:51:04,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664233 2024-11-19T17:51:04,238 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,238 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,238 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,238 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,238 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,239 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664226 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664233 2024-11-19T17:51:04,239 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,239 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,239 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,240 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1149, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,240 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,240 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664233) roll requested 2024-11-19T17:51:04,240 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664240 2024-11-19T17:51:04,246 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,246 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664233 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664240 2024-11-19T17:51:04,247 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,247 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,247 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,247 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1150, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,247 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,247 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664240) roll requested 2024-11-19T17:51:04,248 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664247 2024-11-19T17:51:04,253 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,253 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,254 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,254 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,254 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,254 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664240 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664247 2024-11-19T17:51:04,254 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,254 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,254 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,255 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1151, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,255 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,255 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664247) roll requested 2024-11-19T17:51:04,255 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664255 2024-11-19T17:51:04,259 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,260 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,260 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,260 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,260 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,260 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664247 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664255 2024-11-19T17:51:04,260 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,261 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,261 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1152, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,261 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664255) roll requested 2024-11-19T17:51:04,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664261 2024-11-19T17:51:04,266 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,266 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,266 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,266 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,266 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,266 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664255 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664261 2024-11-19T17:51:04,267 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,267 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,267 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,267 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1153, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,267 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,267 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664261) roll requested 2024-11-19T17:51:04,267 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664267 2024-11-19T17:51:04,271 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,271 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,272 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,272 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,272 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,272 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664261 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664267 2024-11-19T17:51:04,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,272 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,273 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,273 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1154, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,273 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664267) roll requested 2024-11-19T17:51:04,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664273 2024-11-19T17:51:04,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,279 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,279 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,279 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,279 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,279 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664267 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664273 2024-11-19T17:51:04,280 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,280 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,280 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,280 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1155, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,280 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,280 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664273) roll requested 2024-11-19T17:51:04,280 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664280 2024-11-19T17:51:04,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,285 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,285 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,285 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,285 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,285 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664273 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664280 2024-11-19T17:51:04,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,285 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,286 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,286 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1156, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,286 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,286 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664280) roll requested 2024-11-19T17:51:04,286 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664286 2024-11-19T17:51:04,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,290 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,290 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,290 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,291 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664280 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664286 2024-11-19T17:51:04,291 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,291 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,291 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,292 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1157, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,292 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664286) roll requested 2024-11-19T17:51:04,292 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664292 2024-11-19T17:51:04,296 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,296 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,296 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,297 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,297 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,297 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664286 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664292 2024-11-19T17:51:04,297 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,297 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,297 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,298 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1158, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,298 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,298 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664292) roll requested 2024-11-19T17:51:04,298 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664298 2024-11-19T17:51:04,302 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,302 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,302 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,302 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,302 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,302 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664292 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664298 2024-11-19T17:51:04,303 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,303 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,303 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,303 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1159, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,303 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664298) roll requested 2024-11-19T17:51:04,304 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664304 2024-11-19T17:51:04,309 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,309 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,309 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,309 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,309 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,309 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664298 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664304 2024-11-19T17:51:04,310 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,310 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,310 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,310 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1160, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,310 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,310 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664304) roll requested 2024-11-19T17:51:04,310 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664310 2024-11-19T17:51:04,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,315 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,315 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,315 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664304 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664310 2024-11-19T17:51:04,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,316 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,316 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,316 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1161, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,316 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,316 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664310) roll requested 2024-11-19T17:51:04,316 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664316 2024-11-19T17:51:04,321 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,321 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,321 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,321 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,321 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664310 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664316 2024-11-19T17:51:04,322 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,322 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,322 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,323 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1162, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,323 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,323 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664316) roll requested 2024-11-19T17:51:04,323 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664323 2024-11-19T17:51:04,327 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,327 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664316 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664323 2024-11-19T17:51:04,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,328 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,328 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,329 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1163, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,329 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664323) roll requested 2024-11-19T17:51:04,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664329 2024-11-19T17:51:04,333 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,334 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,334 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,334 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,334 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664323 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664329 2024-11-19T17:51:04,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,334 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,335 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1164, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,335 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,335 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664329) roll requested 2024-11-19T17:51:04,335 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664335 2024-11-19T17:51:04,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,339 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,339 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,339 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,339 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,339 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664329 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664335 2024-11-19T17:51:04,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,340 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,340 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1165, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,340 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,340 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664335) roll requested 2024-11-19T17:51:04,340 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664340 2024-11-19T17:51:04,345 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,345 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,345 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,345 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,345 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,345 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664335 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664340 2024-11-19T17:51:04,346 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,346 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,346 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,346 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1166, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,346 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,346 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664340) roll requested 2024-11-19T17:51:04,346 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664346 2024-11-19T17:51:04,351 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,351 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,351 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,351 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,351 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,351 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664340 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664346 2024-11-19T17:51:04,352 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,352 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,352 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,352 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1167, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,352 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,352 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664346) roll requested 2024-11-19T17:51:04,353 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664352 2024-11-19T17:51:04,357 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,357 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,357 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,357 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,357 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,357 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664346 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664352 2024-11-19T17:51:04,358 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,358 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,358 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,358 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1168, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,358 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,358 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664352) roll requested 2024-11-19T17:51:04,359 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664359 2024-11-19T17:51:04,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,363 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,363 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,363 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,363 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,363 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664352 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664359 2024-11-19T17:51:04,365 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,365 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,366 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,366 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1169, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,366 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,366 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664359) roll requested 2024-11-19T17:51:04,366 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664366 2024-11-19T17:51:04,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,371 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664359 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664366 2024-11-19T17:51:04,371 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,371 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,372 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1170, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,372 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,372 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664366) roll requested 2024-11-19T17:51:04,372 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664372 2024-11-19T17:51:04,377 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,377 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664366 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664372 2024-11-19T17:51:04,378 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,378 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,378 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,378 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1171, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,378 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,378 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664372) roll requested 2024-11-19T17:51:04,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664378 2024-11-19T17:51:04,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,383 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,383 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664372 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664378 2024-11-19T17:51:04,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,384 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,384 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1172, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,384 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664378) roll requested 2024-11-19T17:51:04,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664384 2024-11-19T17:51:04,390 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,390 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,390 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,390 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,390 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,390 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664378 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664384 2024-11-19T17:51:04,391 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,391 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,391 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,391 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1173, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,391 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,391 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664384) roll requested 2024-11-19T17:51:04,391 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664391 2024-11-19T17:51:04,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,396 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,396 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664384 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664391 2024-11-19T17:51:04,397 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,397 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,397 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1174, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,397 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664391) roll requested 2024-11-19T17:51:04,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664397 2024-11-19T17:51:04,404 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,404 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,405 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664391 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664397 2024-11-19T17:51:04,418 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,418 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,419 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,419 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1175, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,419 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,419 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664397) roll requested 2024-11-19T17:51:04,419 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664419 2024-11-19T17:51:04,425 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,425 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,426 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,426 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664397 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664419 2024-11-19T17:51:04,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,432 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,432 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1176, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,433 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664419) roll requested 2024-11-19T17:51:04,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664433 2024-11-19T17:51:04,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,440 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664419 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664433 2024-11-19T17:51:04,444 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,444 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,444 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,445 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1177, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,445 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664433) roll requested 2024-11-19T17:51:04,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664445 2024-11-19T17:51:04,453 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,453 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,453 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,453 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,453 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,453 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664433 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664445 2024-11-19T17:51:04,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-19T17:51:04,456 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-19T17:51:04,457 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1178, max=55; forcing (partial) flush of 1 region(s): c82f2c332c5429d4b3c9bb758165d0bc[MyCF] 2024-11-19T17:51:04,457 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of c82f2c332c5429d4b3c9bb758165d0bc, because it is not online on us 2024-11-19T17:51:04,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732038664445) roll requested 2024-11-19T17:51:04,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732038664457 2024-11-19T17:51:04,458 ERROR [Time-limited test {}] wal.AbstractFSWAL(939): Failed log archiving for the log /a/b, java.io.IOException: throw archival exception at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.archiveLogFile(TestFailedAppendAndSync.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.archive(AbstractFSWAL.java:934) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.archive(TestFailedAppendAndSync.java:143) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync.testLockupAroundBadAssignSync(TestFailedAppendAndSync.java:280) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:51:04,460 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c82f2c332c5429d4b3c9bb758165d0bc, disabling compactions & flushes 2024-11-19T17:51:04,460 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testLockupAroundBadAssignSync,,1732038634069.c82f2c332c5429d4b3c9bb758165d0bc. 2024-11-19T17:51:04,461 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testLockupAroundBadAssignSync,,1732038634069.c82f2c332c5429d4b3c9bb758165d0bc. 2024-11-19T17:51:04,461 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testLockupAroundBadAssignSync,,1732038634069.c82f2c332c5429d4b3c9bb758165d0bc. after waiting 0 ms 2024-11-19T17:51:04,461 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testLockupAroundBadAssignSync,,1732038634069.c82f2c332c5429d4b3c9bb758165d0bc. 2024-11-19T17:51:04,462 WARN [LogRoller {}] wal.AbstractProtobufLogWriter(199): Init output failed, path=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732038664457 java.io.InterruptedIOException: java.lang.InterruptedException at org.apache.hadoop.util.Shell.runCommand(Shell.java:1071) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.run(Shell.java:959) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1282) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.execCommand(Shell.java:1377) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.execCommand(Shell.java:1359) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.setPermission(RawLocalFileSystem.java:1116) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem$LocalFSFileOutputStream.(RawLocalFileSystem.java:454) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem$LocalFSFileOutputStream.(RawLocalFileSystem.java:414) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.createOutputStreamWithMode(RawLocalFileSystem.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.create(RawLocalFileSystem.java:566) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.create(RawLocalFileSystem.java:597) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.ChecksumFileSystem$ChecksumFSOutputSummer.(ChecksumFileSystem.java:633) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.ChecksumFileSystem.create(ChecksumFileSystem.java:721) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.ChecksumFileSystem.createNonRecursive(ChecksumFileSystem.java:765) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystem$FileSystemDataOutputStreamBuilder.build(FileSystem.java:4705) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufLogWriter.initOutput(ProtobufLogWriter.java:117) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufLogWriter.init(AbstractProtobufLogWriter.java:171) ~[classes/:?] at org.apache.hadoop.hbase.wal.FSHLogProvider.createWriter(FSHLogProvider.java:82) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.createWriterInstance(FSHLog.java:259) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.createWriterInstance(TestFailedAppendAndSync.java:148) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.createWriterInstance(TestFailedAppendAndSync.java:114) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.rollWriterInternal(AbstractFSWAL.java:1099) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$rollWriter$9(AbstractFSWAL.java:1136) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.rollWriter(AbstractFSWAL.java:1136) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.rollWriter(TestFailedAppendAndSync.java:129) ~[test-classes/:?] at org.apache.hadoop.hbase.wal.AbstractWALRoller$RollController.rollWal(AbstractWALRoller.java:311) ~[classes/:?] at org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:212) ~[classes/:?] Caused by: java.lang.InterruptedException at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at java.lang.ProcessImpl.waitFor(ProcessImpl.java:434) ~[?:?] at org.apache.hadoop.util.Shell.runCommand(Shell.java:1061) ~[hadoop-common-3.4.1.jar:?] ... 27 more 2024-11-19T17:51:04,462 DEBUG [LogRoller {}] wal.FSHLogProvider(93): Error instantiating log writer. java.io.InterruptedIOException: java.lang.InterruptedException at org.apache.hadoop.util.Shell.runCommand(Shell.java:1071) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.run(Shell.java:959) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1282) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.execCommand(Shell.java:1377) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.execCommand(Shell.java:1359) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.setPermission(RawLocalFileSystem.java:1116) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem$LocalFSFileOutputStream.(RawLocalFileSystem.java:454) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem$LocalFSFileOutputStream.(RawLocalFileSystem.java:414) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.createOutputStreamWithMode(RawLocalFileSystem.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.create(RawLocalFileSystem.java:566) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.create(RawLocalFileSystem.java:597) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.ChecksumFileSystem$ChecksumFSOutputSummer.(ChecksumFileSystem.java:633) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.ChecksumFileSystem.create(ChecksumFileSystem.java:721) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.ChecksumFileSystem.createNonRecursive(ChecksumFileSystem.java:765) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystem$FileSystemDataOutputStreamBuilder.build(FileSystem.java:4705) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufLogWriter.initOutput(ProtobufLogWriter.java:117) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufLogWriter.init(AbstractProtobufLogWriter.java:171) ~[classes/:?] at org.apache.hadoop.hbase.wal.FSHLogProvider.createWriter(FSHLogProvider.java:82) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.createWriterInstance(FSHLog.java:259) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.createWriterInstance(TestFailedAppendAndSync.java:148) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.createWriterInstance(TestFailedAppendAndSync.java:114) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.rollWriterInternal(AbstractFSWAL.java:1099) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$rollWriter$9(AbstractFSWAL.java:1136) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.rollWriter(AbstractFSWAL.java:1136) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.rollWriter(TestFailedAppendAndSync.java:129) ~[test-classes/:?] at org.apache.hadoop.hbase.wal.AbstractWALRoller$RollController.rollWal(AbstractWALRoller.java:311) ~[classes/:?] at org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:212) ~[classes/:?] Caused by: java.lang.InterruptedException at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at java.lang.ProcessImpl.waitFor(ProcessImpl.java:434) ~[?:?] at org.apache.hadoop.util.Shell.runCommand(Shell.java:1061) ~[hadoop-common-3.4.1.jar:?] ... 27 more 2024-11-19T17:51:04,462 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 83 in region testLockupAroundBadAssignSync,,1732038634069.c82f2c332c5429d4b3c9bb758165d0bc. 2024-11-19T17:51:04,462 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testLockupAroundBadAssignSync,,1732038634069.c82f2c332c5429d4b3c9bb758165d0bc. 2024-11-19T17:51:04,463 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c82f2c332c5429d4b3c9bb758165d0bc: Waiting for close lock at 1732038664460Disabling compacts and flushes for region at 1732038664460Disabling writes for close at 1732038664461 (+1 ms)Writing region close event to WAL at 1732038664462 (+1 ms)Closed at 1732038664462 2024-11-19T17:51:04,462 ERROR [LogRoller {}] wal.AbstractWALRoller(227): Roll wal failed and waiting timeout, will not retry java.io.IOException: cannot get log writer at org.apache.hadoop.hbase.wal.FSHLogProvider.createWriter(FSHLogProvider.java:95) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.createWriterInstance(FSHLog.java:259) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.createWriterInstance(TestFailedAppendAndSync.java:148) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.createWriterInstance(TestFailedAppendAndSync.java:114) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.rollWriterInternal(AbstractFSWAL.java:1099) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$rollWriter$9(AbstractFSWAL.java:1136) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.rollWriter(AbstractFSWAL.java:1136) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.rollWriter(TestFailedAppendAndSync.java:129) ~[test-classes/:?] at org.apache.hadoop.hbase.wal.AbstractWALRoller$RollController.rollWal(AbstractWALRoller.java:311) ~[classes/:?] at org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:212) ~[classes/:?] Caused by: java.io.InterruptedIOException: java.lang.InterruptedException at org.apache.hadoop.util.Shell.runCommand(Shell.java:1071) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.run(Shell.java:959) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1282) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.execCommand(Shell.java:1377) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.execCommand(Shell.java:1359) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.setPermission(RawLocalFileSystem.java:1116) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem$LocalFSFileOutputStream.(RawLocalFileSystem.java:454) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem$LocalFSFileOutputStream.(RawLocalFileSystem.java:414) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.createOutputStreamWithMode(RawLocalFileSystem.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.create(RawLocalFileSystem.java:566) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.create(RawLocalFileSystem.java:597) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.ChecksumFileSystem$ChecksumFSOutputSummer.(ChecksumFileSystem.java:633) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.ChecksumFileSystem.create(ChecksumFileSystem.java:721) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.ChecksumFileSystem.createNonRecursive(ChecksumFileSystem.java:765) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystem$FileSystemDataOutputStreamBuilder.build(FileSystem.java:4705) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufLogWriter.initOutput(ProtobufLogWriter.java:117) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufLogWriter.init(AbstractProtobufLogWriter.java:171) ~[classes/:?] at org.apache.hadoop.hbase.wal.FSHLogProvider.createWriter(FSHLogProvider.java:82) ~[classes/:?] ... 10 more Caused by: java.lang.InterruptedException at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at java.lang.ProcessImpl.waitFor(ProcessImpl.java:434) ~[?:?] at org.apache.hadoop.util.Shell.runCommand(Shell.java:1061) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.run(Shell.java:959) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1282) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.execCommand(Shell.java:1377) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.execCommand(Shell.java:1359) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.setPermission(RawLocalFileSystem.java:1116) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem$LocalFSFileOutputStream.(RawLocalFileSystem.java:454) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem$LocalFSFileOutputStream.(RawLocalFileSystem.java:414) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.createOutputStreamWithMode(RawLocalFileSystem.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.create(RawLocalFileSystem.java:566) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.create(RawLocalFileSystem.java:597) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.ChecksumFileSystem$ChecksumFSOutputSummer.(ChecksumFileSystem.java:633) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.ChecksumFileSystem.create(ChecksumFileSystem.java:721) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.ChecksumFileSystem.createNonRecursive(ChecksumFileSystem.java:765) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystem$FileSystemDataOutputStreamBuilder.build(FileSystem.java:4705) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufLogWriter.initOutput(ProtobufLogWriter.java:117) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufLogWriter.init(AbstractProtobufLogWriter.java:171) ~[classes/:?] at org.apache.hadoop.hbase.wal.FSHLogProvider.createWriter(FSHLogProvider.java:82) ~[classes/:?] ... 10 more 2024-11-19T17:51:04,463 INFO [LogRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-19T17:51:04,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,464 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-19T17:51:04,707 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: DodgyFSLog wal:(num 1732038664457) 2024-11-19T17:51:04,708 INFO [Time-limited test {}] regionserver.TestFailedAppendAndSync(105): Cleaning test directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cdcfa2bd-1303-b7c2-42e0-e6f2e5e6b673 2024-11-19T17:51:05,132 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.TestFailedAppendAndSync#testLockupAroundBadAssignSync Thread=22 (was 11) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: process reaper java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=287 (was 287), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=561 (was 627), ProcessCount=11 (was 11), AvailableMemoryMB=9435 (was 10187)